Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Cadence USBSS DRD Driver - gadget side.
4 *
5 * Copyright (C) 2018-2019 Cadence Design Systems.
6 * Copyright (C) 2017-2018 NXP
7 *
8 * Authors: Pawel Jez <pjez@cadence.com>,
9 * Pawel Laszczak <pawell@cadence.com>
10 * Peter Chen <peter.chen@nxp.com>
11 */
12
13/*
14 * Work around 1:
15 * At some situations, the controller may get stale data address in TRB
16 * at below sequences:
17 * 1. Controller read TRB includes data address
18 * 2. Software updates TRBs includes data address and Cycle bit
19 * 3. Controller read TRB which includes Cycle bit
20 * 4. DMA run with stale data address
21 *
22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
23 * After preparing all TRBs driver needs to check the position of DMA and
24 * if the DMA point to the first just added TRB and doorbell is 1,
25 * then driver must defer making this TRB as valid. This TRB will be make
26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
27 * interrupt.
28 *
29 * Issue has been fixed in DEV_VER_V3 version of controller.
30 *
31 * Work around 2:
32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
34 * in correct order. If the first packet in the buffer will not be handled,
35 * then the following packets directed for other endpoints and functions
36 * will be blocked.
37 * Additionally the packets directed to one endpoint can block entire on-chip
38 * buffers. In this case transfer to other endpoints also will blocked.
39 *
40 * To resolve this issue after raising the descriptor missing interrupt
41 * driver prepares internal usb_request object and use it to arm DMA transfer.
42 *
43 * The problematic situation was observed in case when endpoint has been enabled
44 * but no usb_request were queued. Driver try detects such endpoints and will
45 * use this workaround only for these endpoint.
46 *
47 * Driver use limited number of buffer. This number can be set by macro
48 * CDNS3_WA2_NUM_BUFFERS.
49 *
50 * Such blocking situation was observed on ACM gadget. For this function
51 * host send OUT data packet but ACM function is not prepared for this packet.
52 * It's cause that buffer placed in on chip memory block transfer to other
53 * endpoints.
54 *
55 * Issue has been fixed in DEV_VER_V2 version of controller.
56 *
57 */
58
59#include <linux/dma-mapping.h>
60#include <linux/usb/gadget.h>
61#include <linux/module.h>
62#include <linux/dmapool.h>
63#include <linux/iopoll.h>
64#include <linux/property.h>
65
66#include "core.h"
67#include "gadget-export.h"
68#include "cdns3-gadget.h"
69#include "cdns3-trace.h"
70#include "drd.h"
71
72static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
73 struct usb_request *request,
74 gfp_t gfp_flags);
75
76static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
77 struct usb_request *request);
78
79static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
80 struct usb_request *request);
81
82/**
83 * cdns3_clear_register_bit - clear bit in given register.
84 * @ptr: address of device controller register to be read and changed
85 * @mask: bits requested to clar
86 */
87static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
88{
89 mask = readl(ptr) & ~mask;
90 writel(mask, ptr);
91}
92
93/**
94 * cdns3_set_register_bit - set bit in given register.
95 * @ptr: address of device controller register to be read and changed
96 * @mask: bits requested to set
97 */
98void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
99{
100 mask = readl(ptr) | mask;
101 writel(mask, ptr);
102}
103
104/**
105 * cdns3_ep_addr_to_index - Macro converts endpoint address to
106 * index of endpoint object in cdns3_device.eps[] container
107 * @ep_addr: endpoint address for which endpoint object is required
108 *
109 */
110u8 cdns3_ep_addr_to_index(u8 ep_addr)
111{
112 return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
113}
114
115static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
116 struct cdns3_endpoint *priv_ep)
117{
118 int dma_index;
119
120 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
121
122 return dma_index / TRB_SIZE;
123}
124
125/**
126 * cdns3_next_request - returns next request from list
127 * @list: list containing requests
128 *
129 * Returns request or NULL if no requests in list
130 */
131struct usb_request *cdns3_next_request(struct list_head *list)
132{
133 return list_first_entry_or_null(list, struct usb_request, list);
134}
135
136/**
137 * cdns3_next_align_buf - returns next buffer from list
138 * @list: list containing buffers
139 *
140 * Returns buffer or NULL if no buffers in list
141 */
142static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
143{
144 return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
145}
146
147/**
148 * cdns3_next_priv_request - returns next request from list
149 * @list: list containing requests
150 *
151 * Returns request or NULL if no requests in list
152 */
153static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
154{
155 return list_first_entry_or_null(list, struct cdns3_request, list);
156}
157
158/**
159 * cdns3_select_ep - selects endpoint
160 * @priv_dev: extended gadget object
161 * @ep: endpoint address
162 */
163void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
164{
165 if (priv_dev->selected_ep == ep)
166 return;
167
168 priv_dev->selected_ep = ep;
169 writel(ep, &priv_dev->regs->ep_sel);
170}
171
172/**
173 * cdns3_get_tdl - gets current tdl for selected endpoint.
174 * @priv_dev: extended gadget object
175 *
176 * Before calling this function the appropriate endpoint must
177 * be selected by means of cdns3_select_ep function.
178 */
179static int cdns3_get_tdl(struct cdns3_device *priv_dev)
180{
181 if (priv_dev->dev_ver < DEV_VER_V3)
182 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
183 else
184 return readl(&priv_dev->regs->ep_tdl);
185}
186
187dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
188 struct cdns3_trb *trb)
189{
190 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
191
192 return priv_ep->trb_pool_dma + offset;
193}
194
195static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
196{
197 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
198
199 if (priv_ep->trb_pool) {
200 dma_pool_free(priv_dev->eps_dma_pool,
201 priv_ep->trb_pool, priv_ep->trb_pool_dma);
202 priv_ep->trb_pool = NULL;
203 }
204}
205
206/**
207 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
208 * @priv_ep: endpoint object
209 *
210 * Function will return 0 on success or -ENOMEM on allocation error
211 */
212int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
213{
214 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
215 int ring_size = TRB_RING_SIZE;
216 int num_trbs = ring_size / TRB_SIZE;
217 struct cdns3_trb *link_trb;
218
219 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
220 cdns3_free_trb_pool(priv_ep);
221
222 if (!priv_ep->trb_pool) {
223 priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,
224 GFP_ATOMIC,
225 &priv_ep->trb_pool_dma);
226
227 if (!priv_ep->trb_pool)
228 return -ENOMEM;
229
230 priv_ep->alloc_ring_size = ring_size;
231 }
232
233 memset(priv_ep->trb_pool, 0, ring_size);
234
235 priv_ep->num_trbs = num_trbs;
236
237 if (!priv_ep->num)
238 return 0;
239
240 /* Initialize the last TRB as Link TRB */
241 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
242
243 if (priv_ep->use_streams) {
244 /*
245 * For stream capable endpoints driver use single correct TRB.
246 * The last trb has zeroed cycle bit
247 */
248 link_trb->control = 0;
249 } else {
250 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
251 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
252 }
253 return 0;
254}
255
256/**
257 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
258 * @priv_ep: endpoint object
259 *
260 * Endpoint must be selected before call to this function
261 */
262static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
263{
264 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
265 int val;
266
267 trace_cdns3_halt(priv_ep, 1, 1);
268
269 writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
270 &priv_dev->regs->ep_cmd);
271
272 /* wait for DFLUSH cleared */
273 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
274 !(val & EP_CMD_DFLUSH), 1, 1000);
275 priv_ep->flags |= EP_STALLED;
276 priv_ep->flags &= ~EP_STALL_PENDING;
277}
278
279/**
280 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
281 * @priv_dev: extended gadget object
282 */
283void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
284{
285 int i;
286
287 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
288
289 cdns3_allow_enable_l1(priv_dev, 0);
290 priv_dev->hw_configured_flag = 0;
291 priv_dev->onchip_used_size = 0;
292 priv_dev->out_mem_is_allocated = 0;
293 priv_dev->wait_for_setup = 0;
294 priv_dev->using_streams = 0;
295
296 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
297 if (priv_dev->eps[i])
298 priv_dev->eps[i]->flags &= ~EP_CONFIGURED;
299}
300
301/**
302 * cdns3_ep_inc_trb - increment a trb index.
303 * @index: Pointer to the TRB index to increment.
304 * @cs: Cycle state
305 * @trb_in_seg: number of TRBs in segment
306 *
307 * The index should never point to the link TRB. After incrementing,
308 * if it is point to the link TRB, wrap around to the beginning and revert
309 * cycle state bit The
310 * link TRB is always at the last TRB entry.
311 */
312static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
313{
314 (*index)++;
315 if (*index == (trb_in_seg - 1)) {
316 *index = 0;
317 *cs ^= 1;
318 }
319}
320
321/**
322 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
323 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
324 */
325static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
326{
327 priv_ep->free_trbs--;
328 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
329}
330
331/**
332 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
333 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
334 */
335static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
336{
337 priv_ep->free_trbs++;
338 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
339}
340
341/**
342 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
343 * @priv_dev: Extended gadget object
344 * @enable: Enable/disable permit to transition to L1.
345 *
346 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
347 * then controller answer with ACK handshake.
348 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
349 * then controller answer with NYET handshake.
350 */
351void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
352{
353 if (enable)
354 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
355 else
356 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
357}
358
359enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
360{
361 u32 reg;
362
363 reg = readl(&priv_dev->regs->usb_sts);
364
365 if (DEV_SUPERSPEED(reg))
366 return USB_SPEED_SUPER;
367 else if (DEV_HIGHSPEED(reg))
368 return USB_SPEED_HIGH;
369 else if (DEV_FULLSPEED(reg))
370 return USB_SPEED_FULL;
371 else if (DEV_LOWSPEED(reg))
372 return USB_SPEED_LOW;
373 return USB_SPEED_UNKNOWN;
374}
375
376/**
377 * cdns3_start_all_request - add to ring all request not started
378 * @priv_dev: Extended gadget object
379 * @priv_ep: The endpoint for whom request will be started.
380 *
381 * Returns return ENOMEM if transfer ring i not enough TRBs to start
382 * all requests.
383 */
384static int cdns3_start_all_request(struct cdns3_device *priv_dev,
385 struct cdns3_endpoint *priv_ep)
386{
387 struct usb_request *request;
388 int ret = 0;
389 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
390
391 /*
392 * If the last pending transfer is INTERNAL
393 * OR streams are enabled for this endpoint
394 * do NOT start new transfer till the last one is pending
395 */
396 if (!pending_empty) {
397 struct cdns3_request *priv_req;
398
399 request = cdns3_next_request(&priv_ep->pending_req_list);
400 priv_req = to_cdns3_request(request);
401 if ((priv_req->flags & REQUEST_INTERNAL) ||
402 (priv_ep->flags & EP_TDLCHK_EN) ||
403 priv_ep->use_streams) {
404 dev_dbg(priv_dev->dev, "Blocking external request\n");
405 return ret;
406 }
407 }
408
409 while (!list_empty(&priv_ep->deferred_req_list)) {
410 request = cdns3_next_request(&priv_ep->deferred_req_list);
411
412 if (!priv_ep->use_streams) {
413 ret = cdns3_ep_run_transfer(priv_ep, request);
414 } else {
415 priv_ep->stream_sg_idx = 0;
416 ret = cdns3_ep_run_stream_transfer(priv_ep, request);
417 }
418 if (ret)
419 return ret;
420
421 list_move_tail(&request->list, &priv_ep->pending_req_list);
422 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
423 break;
424 }
425
426 priv_ep->flags &= ~EP_RING_FULL;
427 return ret;
428}
429
430/*
431 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
432 * driver try to detect whether endpoint need additional internal
433 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
434 * if before first DESCMISS interrupt the DMA will be armed.
435 */
436#define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
437 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
438 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
439 (reg) |= EP_STS_EN_DESCMISEN; \
440 } } while (0)
441
442static void __cdns3_descmiss_copy_data(struct usb_request *request,
443 struct usb_request *descmiss_req)
444{
445 int length = request->actual + descmiss_req->actual;
446 struct scatterlist *s = request->sg;
447
448 if (!s) {
449 if (length <= request->length) {
450 memcpy(&((u8 *)request->buf)[request->actual],
451 descmiss_req->buf,
452 descmiss_req->actual);
453 request->actual = length;
454 } else {
455 /* It should never occures */
456 request->status = -ENOMEM;
457 }
458 } else {
459 if (length <= sg_dma_len(s)) {
460 void *p = phys_to_virt(sg_dma_address(s));
461
462 memcpy(&((u8 *)p)[request->actual],
463 descmiss_req->buf,
464 descmiss_req->actual);
465 request->actual = length;
466 } else {
467 request->status = -ENOMEM;
468 }
469 }
470}
471
472/**
473 * cdns3_wa2_descmiss_copy_data - copy data from internal requests to
474 * request queued by class driver.
475 * @priv_ep: extended endpoint object
476 * @request: request object
477 */
478static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
479 struct usb_request *request)
480{
481 struct usb_request *descmiss_req;
482 struct cdns3_request *descmiss_priv_req;
483
484 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
485 int chunk_end;
486
487 descmiss_priv_req =
488 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
489 descmiss_req = &descmiss_priv_req->request;
490
491 /* driver can't touch pending request */
492 if (descmiss_priv_req->flags & REQUEST_PENDING)
493 break;
494
495 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
496 request->status = descmiss_req->status;
497 __cdns3_descmiss_copy_data(request, descmiss_req);
498 list_del_init(&descmiss_priv_req->list);
499 kfree(descmiss_req->buf);
500 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
501 --priv_ep->wa2_counter;
502
503 if (!chunk_end)
504 break;
505 }
506}
507
508static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
509 struct cdns3_endpoint *priv_ep,
510 struct cdns3_request *priv_req)
511{
512 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
513 priv_req->flags & REQUEST_INTERNAL) {
514 struct usb_request *req;
515
516 req = cdns3_next_request(&priv_ep->deferred_req_list);
517
518 priv_ep->descmis_req = NULL;
519
520 if (!req)
521 return NULL;
522
523 /* unmap the gadget request before copying data */
524 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
525 priv_ep->dir);
526
527 cdns3_wa2_descmiss_copy_data(priv_ep, req);
528 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
529 req->length != req->actual) {
530 /* wait for next part of transfer */
531 /* re-map the gadget request buffer*/
532 usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
533 usb_endpoint_dir_in(priv_ep->endpoint.desc));
534 return NULL;
535 }
536
537 if (req->status == -EINPROGRESS)
538 req->status = 0;
539
540 list_del_init(&req->list);
541 cdns3_start_all_request(priv_dev, priv_ep);
542 return req;
543 }
544
545 return &priv_req->request;
546}
547
548static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
549 struct cdns3_endpoint *priv_ep,
550 struct cdns3_request *priv_req)
551{
552 int deferred = 0;
553
554 /*
555 * If transfer was queued before DESCMISS appear than we
556 * can disable handling of DESCMISS interrupt. Driver assumes that it
557 * can disable special treatment for this endpoint.
558 */
559 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
560 u32 reg;
561
562 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
563 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
564 reg = readl(&priv_dev->regs->ep_sts_en);
565 reg &= ~EP_STS_EN_DESCMISEN;
566 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
567 writel(reg, &priv_dev->regs->ep_sts_en);
568 }
569
570 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
571 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
572 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
573
574 /*
575 * DESCMISS transfer has been finished, so data will be
576 * directly copied from internal allocated usb_request
577 * objects.
578 */
579 if (pending_empty && !descmiss_empty &&
580 !(priv_req->flags & REQUEST_INTERNAL)) {
581 cdns3_wa2_descmiss_copy_data(priv_ep,
582 &priv_req->request);
583
584 trace_cdns3_wa2(priv_ep, "get internal stored data");
585
586 list_add_tail(&priv_req->request.list,
587 &priv_ep->pending_req_list);
588 cdns3_gadget_giveback(priv_ep, priv_req,
589 priv_req->request.status);
590
591 /*
592 * Intentionally driver returns positive value as
593 * correct value. It informs that transfer has
594 * been finished.
595 */
596 return EINPROGRESS;
597 }
598
599 /*
600 * Driver will wait for completion DESCMISS transfer,
601 * before starts new, not DESCMISS transfer.
602 */
603 if (!pending_empty && !descmiss_empty) {
604 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
605 deferred = 1;
606 }
607
608 if (priv_req->flags & REQUEST_INTERNAL)
609 list_add_tail(&priv_req->list,
610 &priv_ep->wa2_descmiss_req_list);
611 }
612
613 return deferred;
614}
615
616static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
617{
618 struct cdns3_request *priv_req;
619
620 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
621 u8 chain;
622
623 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
624 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
625
626 trace_cdns3_wa2(priv_ep, "removes eldest request");
627
628 kfree(priv_req->request.buf);
629 list_del_init(&priv_req->list);
630 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
631 &priv_req->request);
632 --priv_ep->wa2_counter;
633
634 if (!chain)
635 break;
636 }
637}
638
639/**
640 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
641 * @priv_ep: extended gadget object
642 *
643 * This function is used only for WA2. For more information see Work around 2
644 * description.
645 */
646static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
647{
648 struct cdns3_request *priv_req;
649 struct usb_request *request;
650 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
651
652 /* check for pending transfer */
653 if (!pending_empty) {
654 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
655 return;
656 }
657
658 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
659 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
660 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
661 }
662
663 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
664
665 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
666 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
667 cdns3_wa2_remove_old_request(priv_ep);
668 }
669
670 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
671 GFP_ATOMIC);
672 if (!request)
673 goto err;
674
675 priv_req = to_cdns3_request(request);
676 priv_req->flags |= REQUEST_INTERNAL;
677
678 /* if this field is still assigned it indicate that transfer related
679 * with this request has not been finished yet. Driver in this
680 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
681 * flag to previous one. It will indicate that current request is
682 * part of the previous one.
683 */
684 if (priv_ep->descmis_req)
685 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
686
687 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
688 GFP_ATOMIC);
689 priv_ep->wa2_counter++;
690
691 if (!priv_req->request.buf) {
692 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
693 goto err;
694 }
695
696 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
697 priv_ep->descmis_req = priv_req;
698
699 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
700 &priv_ep->descmis_req->request,
701 GFP_ATOMIC);
702
703 return;
704
705err:
706 dev_err(priv_ep->cdns3_dev->dev,
707 "Failed: No sufficient memory for DESCMIS\n");
708}
709
710static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
711{
712 u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
713
714 if (tdl) {
715 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
716
717 writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
718 &priv_dev->regs->ep_cmd);
719 }
720}
721
722static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
723{
724 u32 ep_sts_reg;
725
726 /* select EP0-out */
727 cdns3_select_ep(priv_dev, 0);
728
729 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
730
731 if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
732 u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
733 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
734
735 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
736 outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
737 u8 pending_empty = list_empty(&outq_ep->pending_req_list);
738
739 if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
740 (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
741 !pending_empty) {
742 } else {
743 u32 ep_sts_en_reg;
744 u32 ep_cmd_reg;
745
746 cdns3_select_ep(priv_dev, outq_ep->num |
747 outq_ep->dir);
748 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
749 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
750
751 outq_ep->flags |= EP_TDLCHK_EN;
752 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
753 EP_CFG_TDL_CHK);
754
755 cdns3_wa2_enable_detection(priv_dev, outq_ep,
756 ep_sts_en_reg);
757 writel(ep_sts_en_reg,
758 &priv_dev->regs->ep_sts_en);
759 /* reset tdl value to zero */
760 cdns3_wa2_reset_tdl(priv_dev);
761 /*
762 * Memory barrier - Reset tdl before ringing the
763 * doorbell.
764 */
765 wmb();
766 if (EP_CMD_DRDY & ep_cmd_reg) {
767 trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
768
769 } else {
770 trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
771 /*
772 * ring doorbell to generate DESCMIS irq
773 */
774 writel(EP_CMD_DRDY,
775 &priv_dev->regs->ep_cmd);
776 }
777 }
778 }
779 }
780}
781
782/**
783 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
784 * @priv_ep: The endpoint to whom the request belongs to
785 * @priv_req: The request we're giving back
786 * @status: completion code for the request
787 *
788 * Must be called with controller's lock held and interrupts disabled. This
789 * function will unmap @req and call its ->complete() callback to notify upper
790 * layers that it has completed.
791 */
792void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
793 struct cdns3_request *priv_req,
794 int status)
795{
796 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
797 struct usb_request *request = &priv_req->request;
798
799 list_del_init(&request->list);
800
801 if (request->status == -EINPROGRESS)
802 request->status = status;
803
804 if (likely(!(priv_req->flags & REQUEST_UNALIGNED)))
805 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
806 priv_ep->dir);
807
808 if ((priv_req->flags & REQUEST_UNALIGNED) &&
809 priv_ep->dir == USB_DIR_OUT && !request->status) {
810 /* Make DMA buffer CPU accessible */
811 dma_sync_single_for_cpu(priv_dev->sysdev,
812 priv_req->aligned_buf->dma,
813 request->actual,
814 priv_req->aligned_buf->dir);
815 memcpy(request->buf, priv_req->aligned_buf->buf,
816 request->actual);
817 }
818
819 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
820 /* All TRBs have finished, clear the counter */
821 priv_req->finished_trb = 0;
822 trace_cdns3_gadget_giveback(priv_req);
823
824 if (priv_dev->dev_ver < DEV_VER_V2) {
825 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
826 priv_req);
827 if (!request)
828 return;
829 }
830
831 /*
832 * zlp request is appended by driver, needn't call usb_gadget_giveback_request() to notify
833 * gadget composite driver.
834 */
835 if (request->complete && request->buf != priv_dev->zlp_buf) {
836 spin_unlock(&priv_dev->lock);
837 usb_gadget_giveback_request(&priv_ep->endpoint,
838 request);
839 spin_lock(&priv_dev->lock);
840 }
841
842 if (request->buf == priv_dev->zlp_buf)
843 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
844}
845
846static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
847{
848 /* Work around for stale data address in TRB*/
849 if (priv_ep->wa1_set) {
850 trace_cdns3_wa1(priv_ep, "restore cycle bit");
851
852 priv_ep->wa1_set = 0;
853 priv_ep->wa1_trb_index = 0xFFFF;
854 if (priv_ep->wa1_cycle_bit) {
855 priv_ep->wa1_trb->control =
856 priv_ep->wa1_trb->control | cpu_to_le32(0x1);
857 } else {
858 priv_ep->wa1_trb->control =
859 priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
860 }
861 }
862}
863
864static void cdns3_free_aligned_request_buf(struct work_struct *work)
865{
866 struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
867 aligned_buf_wq);
868 struct cdns3_aligned_buf *buf, *tmp;
869 unsigned long flags;
870
871 spin_lock_irqsave(&priv_dev->lock, flags);
872
873 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
874 if (!buf->in_use) {
875 list_del(&buf->list);
876
877 /*
878 * Re-enable interrupts to free DMA capable memory.
879 * Driver can't free this memory with disabled
880 * interrupts.
881 */
882 spin_unlock_irqrestore(&priv_dev->lock, flags);
883 dma_free_noncoherent(priv_dev->sysdev, buf->size,
884 buf->buf, buf->dma, buf->dir);
885 kfree(buf);
886 spin_lock_irqsave(&priv_dev->lock, flags);
887 }
888 }
889
890 spin_unlock_irqrestore(&priv_dev->lock, flags);
891}
892
893static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
894{
895 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
896 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
897 struct cdns3_aligned_buf *buf;
898
899 /* check if buffer is aligned to 8. */
900 if (!((uintptr_t)priv_req->request.buf & 0x7))
901 return 0;
902
903 buf = priv_req->aligned_buf;
904
905 if (!buf || priv_req->request.length > buf->size) {
906 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
907 if (!buf)
908 return -ENOMEM;
909
910 buf->size = priv_req->request.length;
911 buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ?
912 DMA_TO_DEVICE : DMA_FROM_DEVICE;
913
914 buf->buf = dma_alloc_noncoherent(priv_dev->sysdev,
915 buf->size,
916 &buf->dma,
917 buf->dir,
918 GFP_ATOMIC);
919 if (!buf->buf) {
920 kfree(buf);
921 return -ENOMEM;
922 }
923
924 if (priv_req->aligned_buf) {
925 trace_cdns3_free_aligned_request(priv_req);
926 priv_req->aligned_buf->in_use = 0;
927 queue_work(system_freezable_wq,
928 &priv_dev->aligned_buf_wq);
929 }
930
931 buf->in_use = 1;
932 priv_req->aligned_buf = buf;
933
934 list_add_tail(&buf->list,
935 &priv_dev->aligned_buf_list);
936 }
937
938 if (priv_ep->dir == USB_DIR_IN) {
939 /* Make DMA buffer CPU accessible */
940 dma_sync_single_for_cpu(priv_dev->sysdev,
941 buf->dma, buf->size, buf->dir);
942 memcpy(buf->buf, priv_req->request.buf,
943 priv_req->request.length);
944 }
945
946 /* Transfer DMA buffer ownership back to device */
947 dma_sync_single_for_device(priv_dev->sysdev,
948 buf->dma, buf->size, buf->dir);
949
950 priv_req->flags |= REQUEST_UNALIGNED;
951 trace_cdns3_prepare_aligned_request(priv_req);
952
953 return 0;
954}
955
956static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
957 struct cdns3_trb *trb)
958{
959 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
960
961 if (!priv_ep->wa1_set) {
962 u32 doorbell;
963
964 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
965
966 if (doorbell) {
967 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
968 priv_ep->wa1_set = 1;
969 priv_ep->wa1_trb = trb;
970 priv_ep->wa1_trb_index = priv_ep->enqueue;
971 trace_cdns3_wa1(priv_ep, "set guard");
972 return 0;
973 }
974 }
975 return 1;
976}
977
978static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
979 struct cdns3_endpoint *priv_ep)
980{
981 int dma_index;
982 u32 doorbell;
983
984 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
985 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
986
987 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
988 cdns3_wa1_restore_cycle_bit(priv_ep);
989}
990
991static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
992 struct usb_request *request)
993{
994 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
995 struct cdns3_request *priv_req;
996 struct cdns3_trb *trb;
997 dma_addr_t trb_dma;
998 int address;
999 u32 control;
1000 u32 length;
1001 u32 tdl;
1002 unsigned int sg_idx = priv_ep->stream_sg_idx;
1003
1004 priv_req = to_cdns3_request(request);
1005 address = priv_ep->endpoint.desc->bEndpointAddress;
1006
1007 priv_ep->flags |= EP_PENDING_REQUEST;
1008
1009 /* must allocate buffer aligned to 8 */
1010 if (priv_req->flags & REQUEST_UNALIGNED)
1011 trb_dma = priv_req->aligned_buf->dma;
1012 else
1013 trb_dma = request->dma;
1014
1015 /* For stream capable endpoints driver use only single TD. */
1016 trb = priv_ep->trb_pool + priv_ep->enqueue;
1017 priv_req->start_trb = priv_ep->enqueue;
1018 priv_req->end_trb = priv_req->start_trb;
1019 priv_req->trb = trb;
1020
1021 cdns3_select_ep(priv_ep->cdns3_dev, address);
1022
1023 control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1024 TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1025
1026 if (!request->num_sgs) {
1027 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1028 length = request->length;
1029 } else {
1030 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
1031 length = request->sg[sg_idx].length;
1032 }
1033
1034 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1035
1036 trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
1037
1038 /*
1039 * For DEV_VER_V2 controller version we have enabled
1040 * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1041 * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1042 */
1043 if (priv_dev->dev_ver >= DEV_VER_V2) {
1044 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1045 trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl));
1046 }
1047 priv_req->flags |= REQUEST_PENDING;
1048
1049 trb->control = cpu_to_le32(control);
1050
1051 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1052
1053 /*
1054 * Memory barrier - Cycle Bit must be set before trb->length and
1055 * trb->buffer fields.
1056 */
1057 wmb();
1058
1059 /* always first element */
1060 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1061 &priv_dev->regs->ep_traddr);
1062
1063 if (!(priv_ep->flags & EP_STALLED)) {
1064 trace_cdns3_ring(priv_ep);
1065 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1066 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1067
1068 priv_ep->prime_flag = false;
1069
1070 /*
1071 * Controller version DEV_VER_V2 tdl calculation
1072 * is based on TRB
1073 */
1074
1075 if (priv_dev->dev_ver < DEV_VER_V2)
1076 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1077 &priv_dev->regs->ep_cmd);
1078 else if (priv_dev->dev_ver > DEV_VER_V2)
1079 writel(tdl, &priv_dev->regs->ep_tdl);
1080
1081 priv_ep->last_stream_id = priv_req->request.stream_id;
1082 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1083 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1084 EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1085
1086 trace_cdns3_doorbell_epx(priv_ep->name,
1087 readl(&priv_dev->regs->ep_traddr));
1088 }
1089
1090 /* WORKAROUND for transition to L0 */
1091 __cdns3_gadget_wakeup(priv_dev);
1092
1093 return 0;
1094}
1095
1096static void cdns3_rearm_drdy_if_needed(struct cdns3_endpoint *priv_ep)
1097{
1098 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1099
1100 if (priv_dev->dev_ver < DEV_VER_V3)
1101 return;
1102
1103 if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) {
1104 writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts);
1105 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1106 }
1107}
1108
1109/**
1110 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1111 * @priv_ep: endpoint object
1112 * @request: request object
1113 *
1114 * Returns zero on success or negative value on failure
1115 */
1116static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1117 struct usb_request *request)
1118{
1119 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1120 struct cdns3_request *priv_req;
1121 struct cdns3_trb *trb;
1122 struct cdns3_trb *link_trb = NULL;
1123 dma_addr_t trb_dma;
1124 u32 togle_pcs = 1;
1125 int sg_iter = 0;
1126 int num_trb_req;
1127 int trb_burst;
1128 int num_trb;
1129 int address;
1130 u32 control;
1131 int pcs;
1132 u16 total_tdl = 0;
1133 struct scatterlist *s = NULL;
1134 bool sg_supported = !!(request->num_mapped_sgs);
1135 u32 ioc = request->no_interrupt ? 0 : TRB_IOC;
1136
1137 num_trb_req = sg_supported ? request->num_mapped_sgs : 1;
1138
1139 /* ISO transfer require each SOF have a TD, each TD include some TRBs */
1140 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1141 num_trb = priv_ep->interval * num_trb_req;
1142 else
1143 num_trb = num_trb_req;
1144
1145 priv_req = to_cdns3_request(request);
1146 address = priv_ep->endpoint.desc->bEndpointAddress;
1147
1148 priv_ep->flags |= EP_PENDING_REQUEST;
1149
1150 /* must allocate buffer aligned to 8 */
1151 if (priv_req->flags & REQUEST_UNALIGNED)
1152 trb_dma = priv_req->aligned_buf->dma;
1153 else
1154 trb_dma = request->dma;
1155
1156 trb = priv_ep->trb_pool + priv_ep->enqueue;
1157 priv_req->start_trb = priv_ep->enqueue;
1158 priv_req->trb = trb;
1159
1160 cdns3_select_ep(priv_ep->cdns3_dev, address);
1161
1162 /* prepare ring */
1163 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
1164 int doorbell, dma_index;
1165 u32 ch_bit = 0;
1166
1167 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1168 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1169
1170 /* Driver can't update LINK TRB if it is current processed. */
1171 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1172 priv_ep->flags |= EP_DEFERRED_DRDY;
1173 return -ENOBUFS;
1174 }
1175
1176 /*updating C bt in Link TRB before starting DMA*/
1177 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1178 /*
1179 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1180 * that DMA stuck at the LINK TRB.
1181 * On the other hand, removing TRB_CHAIN for longer TRs for
1182 * epXout cause that DMA stuck after handling LINK TRB.
1183 * To eliminate this strange behavioral driver set TRB_CHAIN
1184 * bit only for TR size > 2.
1185 */
1186 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1187 TRBS_PER_SEGMENT > 2)
1188 ch_bit = TRB_CHAIN;
1189
1190 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1191 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
1192
1193 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1194 /*
1195 * ISO require LINK TRB must be first one of TD.
1196 * Fill LINK TRBs for left trb space to simply software process logic.
1197 */
1198 while (priv_ep->enqueue) {
1199 *trb = *link_trb;
1200 trace_cdns3_prepare_trb(priv_ep, trb);
1201
1202 cdns3_ep_inc_enq(priv_ep);
1203 trb = priv_ep->trb_pool + priv_ep->enqueue;
1204 priv_req->trb = trb;
1205 }
1206 }
1207 }
1208
1209 if (num_trb > priv_ep->free_trbs) {
1210 priv_ep->flags |= EP_RING_FULL;
1211 return -ENOBUFS;
1212 }
1213
1214 if (priv_dev->dev_ver <= DEV_VER_V2)
1215 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1216
1217 /* set incorrect Cycle Bit for first trb*/
1218 control = priv_ep->pcs ? 0 : TRB_CYCLE;
1219 trb->length = 0;
1220 if (priv_dev->dev_ver >= DEV_VER_V2) {
1221 u16 td_size;
1222
1223 td_size = DIV_ROUND_UP(request->length,
1224 priv_ep->endpoint.maxpacket);
1225 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1226 trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
1227 else
1228 control |= TRB_TDL_HS_SIZE(td_size);
1229 }
1230
1231 do {
1232 u32 length;
1233
1234 if (!(sg_iter % num_trb_req) && sg_supported)
1235 s = request->sg;
1236
1237 /* fill TRB */
1238 control |= TRB_TYPE(TRB_NORMAL);
1239 if (sg_supported) {
1240 trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
1241 length = sg_dma_len(s);
1242 } else {
1243 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1244 length = request->length;
1245 }
1246
1247 if (priv_ep->flags & EP_TDLCHK_EN)
1248 total_tdl += DIV_ROUND_UP(length,
1249 priv_ep->endpoint.maxpacket);
1250
1251 trb_burst = priv_ep->trb_burst_size;
1252
1253 /*
1254 * Supposed DMA cross 4k bounder problem should be fixed at DEV_VER_V2, but still
1255 * met problem when do ISO transfer if sg enabled.
1256 *
1257 * Data pattern likes below when sg enabled, package size is 1k and mult is 2
1258 * [UVC Header(8B) ] [data(3k - 8)] ...
1259 *
1260 * The received data at offset 0xd000 will get 0xc000 data, len 0x70. Error happen
1261 * as below pattern:
1262 * 0xd000: wrong
1263 * 0xe000: wrong
1264 * 0xf000: correct
1265 * 0x10000: wrong
1266 * 0x11000: wrong
1267 * 0x12000: correct
1268 * ...
1269 *
1270 * But it is still unclear about why error have not happen below 0xd000, it should
1271 * cross 4k bounder. But anyway, the below code can fix this problem.
1272 *
1273 * To avoid DMA cross 4k bounder at ISO transfer, reduce burst len according to 16.
1274 */
1275 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_dev->dev_ver <= DEV_VER_V2)
1276 if (ALIGN_DOWN(trb->buffer, SZ_4K) !=
1277 ALIGN_DOWN(trb->buffer + length, SZ_4K))
1278 trb_burst = 16;
1279
1280 trb->length |= cpu_to_le32(TRB_BURST_LEN(trb_burst) |
1281 TRB_LEN(length));
1282 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1283
1284 /*
1285 * first trb should be prepared as last to avoid processing
1286 * transfer to early
1287 */
1288 if (sg_iter != 0)
1289 control |= pcs;
1290
1291 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
1292 control |= ioc | TRB_ISP;
1293 } else {
1294 /* for last element in TD or in SG list */
1295 if (sg_iter == (num_trb - 1) && sg_iter != 0)
1296 control |= pcs | ioc | TRB_ISP;
1297 }
1298
1299 if (sg_iter)
1300 trb->control = cpu_to_le32(control);
1301 else
1302 priv_req->trb->control = cpu_to_le32(control);
1303
1304 if (sg_supported) {
1305 trb->control |= cpu_to_le32(TRB_ISP);
1306 /* Don't set chain bit for last TRB */
1307 if ((sg_iter % num_trb_req) < num_trb_req - 1)
1308 trb->control |= cpu_to_le32(TRB_CHAIN);
1309
1310 s = sg_next(s);
1311 }
1312
1313 control = 0;
1314 ++sg_iter;
1315 priv_req->end_trb = priv_ep->enqueue;
1316 cdns3_ep_inc_enq(priv_ep);
1317 trb = priv_ep->trb_pool + priv_ep->enqueue;
1318 trb->length = 0;
1319 } while (sg_iter < num_trb);
1320
1321 trb = priv_req->trb;
1322
1323 priv_req->flags |= REQUEST_PENDING;
1324 priv_req->num_of_trb = num_trb;
1325
1326 if (sg_iter == 1)
1327 trb->control |= cpu_to_le32(ioc | TRB_ISP);
1328
1329 if (priv_dev->dev_ver < DEV_VER_V2 &&
1330 (priv_ep->flags & EP_TDLCHK_EN)) {
1331 u16 tdl = total_tdl;
1332 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1333
1334 if (tdl > EP_CMD_TDL_MAX) {
1335 tdl = EP_CMD_TDL_MAX;
1336 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1337 }
1338
1339 if (old_tdl < tdl) {
1340 tdl -= old_tdl;
1341 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1342 &priv_dev->regs->ep_cmd);
1343 }
1344 }
1345
1346 /*
1347 * Memory barrier - cycle bit must be set before other filds in trb.
1348 */
1349 wmb();
1350
1351 /* give the TD to the consumer*/
1352 if (togle_pcs)
1353 trb->control = trb->control ^ cpu_to_le32(1);
1354
1355 if (priv_dev->dev_ver <= DEV_VER_V2)
1356 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1357
1358 if (num_trb > 1) {
1359 int i = 0;
1360
1361 while (i < num_trb) {
1362 trace_cdns3_prepare_trb(priv_ep, trb + i);
1363 if (trb + i == link_trb) {
1364 trb = priv_ep->trb_pool;
1365 num_trb = num_trb - i;
1366 i = 0;
1367 } else {
1368 i++;
1369 }
1370 }
1371 } else {
1372 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1373 }
1374
1375 /*
1376 * Memory barrier - Cycle Bit must be set before trb->length and
1377 * trb->buffer fields.
1378 */
1379 wmb();
1380
1381 /*
1382 * For DMULT mode we can set address to transfer ring only once after
1383 * enabling endpoint.
1384 */
1385 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1386 /*
1387 * Until SW is not ready to handle the OUT transfer the ISO OUT
1388 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1389 * EP_CFG_ENABLE must be set before updating ep_traddr.
1390 */
1391 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
1392 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1393 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1394 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1395 EP_CFG_ENABLE);
1396 }
1397
1398 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1399 priv_req->start_trb * TRB_SIZE),
1400 &priv_dev->regs->ep_traddr);
1401
1402 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1403 }
1404
1405 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1406 trace_cdns3_ring(priv_ep);
1407 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1408 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1409 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1410 cdns3_rearm_drdy_if_needed(priv_ep);
1411 trace_cdns3_doorbell_epx(priv_ep->name,
1412 readl(&priv_dev->regs->ep_traddr));
1413 }
1414
1415 /* WORKAROUND for transition to L0 */
1416 __cdns3_gadget_wakeup(priv_dev);
1417
1418 return 0;
1419}
1420
1421void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1422{
1423 struct cdns3_endpoint *priv_ep;
1424 struct usb_ep *ep;
1425
1426 if (priv_dev->hw_configured_flag)
1427 return;
1428
1429 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1430
1431 cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1432 USB_CONF_U1EN | USB_CONF_U2EN);
1433
1434 priv_dev->hw_configured_flag = 1;
1435
1436 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1437 if (ep->enabled) {
1438 priv_ep = ep_to_cdns3_ep(ep);
1439 cdns3_start_all_request(priv_dev, priv_ep);
1440 }
1441 }
1442
1443 cdns3_allow_enable_l1(priv_dev, 1);
1444}
1445
1446/**
1447 * cdns3_trb_handled - check whether trb has been handled by DMA
1448 *
1449 * @priv_ep: extended endpoint object.
1450 * @priv_req: request object for checking
1451 *
1452 * Endpoint must be selected before invoking this function.
1453 *
1454 * Returns false if request has not been handled by DMA, else returns true.
1455 *
1456 * SR - start ring
1457 * ER - end ring
1458 * DQ = priv_ep->dequeue - dequeue position
1459 * EQ = priv_ep->enqueue - enqueue position
1460 * ST = priv_req->start_trb - index of first TRB in transfer ring
1461 * ET = priv_req->end_trb - index of last TRB in transfer ring
1462 * CI = current_index - index of processed TRB by DMA.
1463 *
1464 * As first step, we check if the TRB between the ST and ET.
1465 * Then, we check if cycle bit for index priv_ep->dequeue
1466 * is correct.
1467 *
1468 * some rules:
1469 * 1. priv_ep->dequeue never equals to current_index.
1470 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1471 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1472 * and priv_ep->free_trbs is zero.
1473 * This case indicate that TR is full.
1474 *
1475 * At below two cases, the request have been handled.
1476 * Case 1 - priv_ep->dequeue < current_index
1477 * SR ... EQ ... DQ ... CI ... ER
1478 * SR ... DQ ... CI ... EQ ... ER
1479 *
1480 * Case 2 - priv_ep->dequeue > current_index
1481 * This situation takes place when CI go through the LINK TRB at the end of
1482 * transfer ring.
1483 * SR ... CI ... EQ ... DQ ... ER
1484 */
1485static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1486 struct cdns3_request *priv_req)
1487{
1488 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1489 struct cdns3_trb *trb;
1490 int current_index = 0;
1491 int handled = 0;
1492 int doorbell;
1493
1494 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1495 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1496
1497 /* current trb doesn't belong to this request */
1498 if (priv_req->start_trb < priv_req->end_trb) {
1499 if (priv_ep->dequeue > priv_req->end_trb)
1500 goto finish;
1501
1502 if (priv_ep->dequeue < priv_req->start_trb)
1503 goto finish;
1504 }
1505
1506 if ((priv_req->start_trb > priv_req->end_trb) &&
1507 (priv_ep->dequeue > priv_req->end_trb) &&
1508 (priv_ep->dequeue < priv_req->start_trb))
1509 goto finish;
1510
1511 if ((priv_req->start_trb == priv_req->end_trb) &&
1512 (priv_ep->dequeue != priv_req->end_trb))
1513 goto finish;
1514
1515 trb = &priv_ep->trb_pool[priv_ep->dequeue];
1516
1517 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1518 goto finish;
1519
1520 if (doorbell == 1 && current_index == priv_ep->dequeue)
1521 goto finish;
1522
1523 /* The corner case for TRBS_PER_SEGMENT equal 2). */
1524 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1525 handled = 1;
1526 goto finish;
1527 }
1528
1529 if (priv_ep->enqueue == priv_ep->dequeue &&
1530 priv_ep->free_trbs == 0) {
1531 handled = 1;
1532 } else if (priv_ep->dequeue < current_index) {
1533 if ((current_index == (priv_ep->num_trbs - 1)) &&
1534 !priv_ep->dequeue)
1535 goto finish;
1536
1537 handled = 1;
1538 } else if (priv_ep->dequeue > current_index) {
1539 handled = 1;
1540 }
1541
1542finish:
1543 trace_cdns3_request_handled(priv_req, current_index, handled);
1544
1545 return handled;
1546}
1547
1548static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1549 struct cdns3_endpoint *priv_ep)
1550{
1551 struct cdns3_request *priv_req;
1552 struct usb_request *request;
1553 struct cdns3_trb *trb;
1554 bool request_handled = false;
1555 bool transfer_end = false;
1556
1557 while (!list_empty(&priv_ep->pending_req_list)) {
1558 request = cdns3_next_request(&priv_ep->pending_req_list);
1559 priv_req = to_cdns3_request(request);
1560
1561 trb = priv_ep->trb_pool + priv_ep->dequeue;
1562
1563 /* The TRB was changed as link TRB, and the request was handled at ep_dequeue */
1564 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
1565
1566 /* ISO ep_traddr may stop at LINK TRB */
1567 if (priv_ep->dequeue == cdns3_get_dma_pos(priv_dev, priv_ep) &&
1568 priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1569 break;
1570
1571 trace_cdns3_complete_trb(priv_ep, trb);
1572 cdns3_ep_inc_deq(priv_ep);
1573 trb = priv_ep->trb_pool + priv_ep->dequeue;
1574 }
1575
1576 if (!request->stream_id) {
1577 /* Re-select endpoint. It could be changed by other CPU
1578 * during handling usb_gadget_giveback_request.
1579 */
1580 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1581
1582 while (cdns3_trb_handled(priv_ep, priv_req)) {
1583 priv_req->finished_trb++;
1584 if (priv_req->finished_trb >= priv_req->num_of_trb)
1585 request_handled = true;
1586
1587 trb = priv_ep->trb_pool + priv_ep->dequeue;
1588 trace_cdns3_complete_trb(priv_ep, trb);
1589
1590 if (!transfer_end)
1591 request->actual +=
1592 TRB_LEN(le32_to_cpu(trb->length));
1593
1594 if (priv_req->num_of_trb > 1 &&
1595 le32_to_cpu(trb->control) & TRB_SMM &&
1596 le32_to_cpu(trb->control) & TRB_CHAIN)
1597 transfer_end = true;
1598
1599 cdns3_ep_inc_deq(priv_ep);
1600 }
1601
1602 if (request_handled) {
1603 /* TRBs are duplicated by priv_ep->interval time for ISO IN */
1604 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_ep->dir)
1605 request->actual /= priv_ep->interval;
1606
1607 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1608 request_handled = false;
1609 transfer_end = false;
1610 } else {
1611 goto prepare_next_td;
1612 }
1613
1614 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1615 TRBS_PER_SEGMENT == 2)
1616 break;
1617 } else {
1618 /* Re-select endpoint. It could be changed by other CPU
1619 * during handling usb_gadget_giveback_request.
1620 */
1621 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1622
1623 trb = priv_ep->trb_pool;
1624 trace_cdns3_complete_trb(priv_ep, trb);
1625
1626 if (trb != priv_req->trb)
1627 dev_warn(priv_dev->dev,
1628 "request_trb=0x%p, queue_trb=0x%p\n",
1629 priv_req->trb, trb);
1630
1631 request->actual += TRB_LEN(le32_to_cpu(trb->length));
1632
1633 if (!request->num_sgs ||
1634 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1635 priv_ep->stream_sg_idx = 0;
1636 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1637 } else {
1638 priv_ep->stream_sg_idx++;
1639 cdns3_ep_run_stream_transfer(priv_ep, request);
1640 }
1641 break;
1642 }
1643 }
1644 priv_ep->flags &= ~EP_PENDING_REQUEST;
1645
1646prepare_next_td:
1647 if (!(priv_ep->flags & EP_STALLED) &&
1648 !(priv_ep->flags & EP_STALL_PENDING))
1649 cdns3_start_all_request(priv_dev, priv_ep);
1650}
1651
1652void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1653{
1654 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1655
1656 cdns3_wa1_restore_cycle_bit(priv_ep);
1657
1658 if (rearm) {
1659 trace_cdns3_ring(priv_ep);
1660
1661 /* Cycle Bit must be updated before arming DMA. */
1662 wmb();
1663 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1664
1665 __cdns3_gadget_wakeup(priv_dev);
1666
1667 trace_cdns3_doorbell_epx(priv_ep->name,
1668 readl(&priv_dev->regs->ep_traddr));
1669 }
1670}
1671
1672static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1673{
1674 u16 tdl = priv_ep->pending_tdl;
1675 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1676
1677 if (tdl > EP_CMD_TDL_MAX) {
1678 tdl = EP_CMD_TDL_MAX;
1679 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1680 } else {
1681 priv_ep->pending_tdl = 0;
1682 }
1683
1684 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1685}
1686
1687/**
1688 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1689 * @priv_ep: endpoint object
1690 *
1691 * Returns 0
1692 */
1693static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1694{
1695 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1696 u32 ep_sts_reg;
1697 struct usb_request *deferred_request;
1698 struct usb_request *pending_request;
1699 u32 tdl = 0;
1700
1701 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1702
1703 trace_cdns3_epx_irq(priv_dev, priv_ep);
1704
1705 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1706 writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1707
1708 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1709 bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1710
1711 tdl = cdns3_get_tdl(priv_dev);
1712
1713 /*
1714 * Continue the previous transfer:
1715 * There is some racing between ERDY and PRIME. The device send
1716 * ERDY and almost in the same time Host send PRIME. It cause
1717 * that host ignore the ERDY packet and driver has to send it
1718 * again.
1719 */
1720 if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) ||
1721 EP_STS_HOSTPP(ep_sts_reg))) {
1722 writel(EP_CMD_ERDY |
1723 EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1724 &priv_dev->regs->ep_cmd);
1725 ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1726 } else {
1727 priv_ep->prime_flag = true;
1728
1729 pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1730 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1731
1732 if (deferred_request && !pending_request) {
1733 cdns3_start_all_request(priv_dev, priv_ep);
1734 }
1735 }
1736 }
1737
1738 if (ep_sts_reg & EP_STS_TRBERR) {
1739 if (priv_ep->flags & EP_STALL_PENDING &&
1740 !(ep_sts_reg & EP_STS_DESCMIS &&
1741 priv_dev->dev_ver < DEV_VER_V2)) {
1742 cdns3_ep_stall_flush(priv_ep);
1743 }
1744
1745 /*
1746 * For isochronous transfer driver completes request on
1747 * IOC or on TRBERR. IOC appears only when device receive
1748 * OUT data packet. If host disable stream or lost some packet
1749 * then the only way to finish all queued transfer is to do it
1750 * on TRBERR event.
1751 */
1752 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1753 !priv_ep->wa1_set) {
1754 if (!priv_ep->dir) {
1755 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1756
1757 ep_cfg &= ~EP_CFG_ENABLE;
1758 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1759 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1760 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
1761 }
1762 cdns3_transfer_completed(priv_dev, priv_ep);
1763 } else if (!(priv_ep->flags & EP_STALLED) &&
1764 !(priv_ep->flags & EP_STALL_PENDING)) {
1765 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1766 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1767 cdns3_start_all_request(priv_dev, priv_ep);
1768 } else {
1769 cdns3_rearm_transfer(priv_ep,
1770 priv_ep->wa1_set);
1771 }
1772 }
1773 }
1774
1775 if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1776 (ep_sts_reg & EP_STS_IOT)) {
1777 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1778 if (ep_sts_reg & EP_STS_ISP)
1779 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1780 else
1781 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1782 }
1783
1784 if (!priv_ep->use_streams) {
1785 if ((ep_sts_reg & EP_STS_IOC) ||
1786 (ep_sts_reg & EP_STS_ISP)) {
1787 cdns3_transfer_completed(priv_dev, priv_ep);
1788 } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1789 priv_ep->pending_tdl) {
1790 /* handle IOT with pending tdl */
1791 cdns3_reprogram_tdl(priv_ep);
1792 }
1793 } else if (priv_ep->dir == USB_DIR_OUT) {
1794 priv_ep->ep_sts_pending |= ep_sts_reg;
1795 } else if (ep_sts_reg & EP_STS_IOT) {
1796 cdns3_transfer_completed(priv_dev, priv_ep);
1797 }
1798 }
1799
1800 /*
1801 * MD_EXIT interrupt sets when stream capable endpoint exits
1802 * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1803 */
1804 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1805 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1806 priv_ep->ep_sts_pending = 0;
1807 cdns3_transfer_completed(priv_dev, priv_ep);
1808 }
1809
1810 /*
1811 * WA2: this condition should only be meet when
1812 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1813 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1814 * In other cases this interrupt will be disabled.
1815 */
1816 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1817 !(priv_ep->flags & EP_STALLED))
1818 cdns3_wa2_descmissing_packet(priv_ep);
1819
1820 return 0;
1821}
1822
1823static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1824{
1825 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)
1826 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1827}
1828
1829/**
1830 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1831 * @priv_dev: extended gadget object
1832 * @usb_ists: bitmap representation of device's reported interrupts
1833 * (usb_ists register value)
1834 */
1835static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1836 u32 usb_ists)
1837__must_hold(&priv_dev->lock)
1838{
1839 int speed = 0;
1840
1841 trace_cdns3_usb_irq(priv_dev, usb_ists);
1842 if (usb_ists & USB_ISTS_L1ENTI) {
1843 /*
1844 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1845 * from L1. To fix it, if any DMA transfer is pending driver
1846 * must starts driving resume signal immediately.
1847 */
1848 if (readl(&priv_dev->regs->drbl))
1849 __cdns3_gadget_wakeup(priv_dev);
1850 }
1851
1852 /* Connection detected */
1853 if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1854 speed = cdns3_get_speed(priv_dev);
1855 priv_dev->gadget.speed = speed;
1856 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1857 cdns3_ep0_config(priv_dev);
1858 }
1859
1860 /* Disconnection detected */
1861 if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1862 spin_unlock(&priv_dev->lock);
1863 cdns3_disconnect_gadget(priv_dev);
1864 spin_lock(&priv_dev->lock);
1865 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1866 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1867 cdns3_hw_reset_eps_config(priv_dev);
1868 }
1869
1870 if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1871 if (priv_dev->gadget_driver &&
1872 priv_dev->gadget_driver->suspend) {
1873 spin_unlock(&priv_dev->lock);
1874 priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1875 spin_lock(&priv_dev->lock);
1876 }
1877 }
1878
1879 if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1880 if (priv_dev->gadget_driver &&
1881 priv_dev->gadget_driver->resume) {
1882 spin_unlock(&priv_dev->lock);
1883 priv_dev->gadget_driver->resume(&priv_dev->gadget);
1884 spin_lock(&priv_dev->lock);
1885 }
1886 }
1887
1888 /* reset*/
1889 if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1890 if (priv_dev->gadget_driver) {
1891 spin_unlock(&priv_dev->lock);
1892 usb_gadget_udc_reset(&priv_dev->gadget,
1893 priv_dev->gadget_driver);
1894 spin_lock(&priv_dev->lock);
1895
1896 /*read again to check the actual speed*/
1897 speed = cdns3_get_speed(priv_dev);
1898 priv_dev->gadget.speed = speed;
1899 cdns3_hw_reset_eps_config(priv_dev);
1900 cdns3_ep0_config(priv_dev);
1901 }
1902 }
1903}
1904
1905/**
1906 * cdns3_device_irq_handler - interrupt handler for device part of controller
1907 *
1908 * @irq: irq number for cdns3 core device
1909 * @data: structure of cdns3
1910 *
1911 * Returns IRQ_HANDLED or IRQ_NONE
1912 */
1913static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1914{
1915 struct cdns3_device *priv_dev = data;
1916 struct cdns *cdns = dev_get_drvdata(priv_dev->dev);
1917 irqreturn_t ret = IRQ_NONE;
1918 u32 reg;
1919
1920 if (cdns->in_lpm)
1921 return ret;
1922
1923 /* check USB device interrupt */
1924 reg = readl(&priv_dev->regs->usb_ists);
1925 if (reg) {
1926 /* After masking interrupts the new interrupts won't be
1927 * reported in usb_ists/ep_ists. In order to not lose some
1928 * of them driver disables only detected interrupts.
1929 * They will be enabled ASAP after clearing source of
1930 * interrupt. This an unusual behavior only applies to
1931 * usb_ists register.
1932 */
1933 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1934 /* mask deferred interrupt. */
1935 writel(reg, &priv_dev->regs->usb_ien);
1936 ret = IRQ_WAKE_THREAD;
1937 }
1938
1939 /* check endpoint interrupt */
1940 reg = readl(&priv_dev->regs->ep_ists);
1941 if (reg) {
1942 writel(0, &priv_dev->regs->ep_ien);
1943 ret = IRQ_WAKE_THREAD;
1944 }
1945
1946 return ret;
1947}
1948
1949/**
1950 * cdns3_device_thread_irq_handler - interrupt handler for device part
1951 * of controller
1952 *
1953 * @irq: irq number for cdns3 core device
1954 * @data: structure of cdns3
1955 *
1956 * Returns IRQ_HANDLED or IRQ_NONE
1957 */
1958static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1959{
1960 struct cdns3_device *priv_dev = data;
1961 irqreturn_t ret = IRQ_NONE;
1962 unsigned long flags;
1963 unsigned int bit;
1964 unsigned long reg;
1965
1966 spin_lock_irqsave(&priv_dev->lock, flags);
1967
1968 reg = readl(&priv_dev->regs->usb_ists);
1969 if (reg) {
1970 writel(reg, &priv_dev->regs->usb_ists);
1971 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1972 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1973 ret = IRQ_HANDLED;
1974 }
1975
1976 reg = readl(&priv_dev->regs->ep_ists);
1977
1978 /* handle default endpoint OUT */
1979 if (reg & EP_ISTS_EP_OUT0) {
1980 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1981 ret = IRQ_HANDLED;
1982 }
1983
1984 /* handle default endpoint IN */
1985 if (reg & EP_ISTS_EP_IN0) {
1986 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1987 ret = IRQ_HANDLED;
1988 }
1989
1990 /* check if interrupt from non default endpoint, if no exit */
1991 reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1992 if (!reg)
1993 goto irqend;
1994
1995 for_each_set_bit(bit, ®,
1996 sizeof(u32) * BITS_PER_BYTE) {
1997 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1998 ret = IRQ_HANDLED;
1999 }
2000
2001 if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
2002 cdns3_wa2_check_outq_status(priv_dev);
2003
2004irqend:
2005 writel(~0, &priv_dev->regs->ep_ien);
2006 spin_unlock_irqrestore(&priv_dev->lock, flags);
2007
2008 return ret;
2009}
2010
2011/**
2012 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
2013 *
2014 * The real reservation will occur during write to EP_CFG register,
2015 * this function is used to check if the 'size' reservation is allowed.
2016 *
2017 * @priv_dev: extended gadget object
2018 * @size: the size (KB) for EP would like to allocate
2019 * @is_in: endpoint direction
2020 *
2021 * Return 0 if the required size can met or negative value on failure
2022 */
2023static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
2024 int size, int is_in)
2025{
2026 int remained;
2027
2028 /* 2KB are reserved for EP0*/
2029 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
2030
2031 if (is_in) {
2032 if (remained < size)
2033 return -EPERM;
2034
2035 priv_dev->onchip_used_size += size;
2036 } else {
2037 int required;
2038
2039 /**
2040 * ALL OUT EPs are shared the same chunk onchip memory, so
2041 * driver checks if it already has assigned enough buffers
2042 */
2043 if (priv_dev->out_mem_is_allocated >= size)
2044 return 0;
2045
2046 required = size - priv_dev->out_mem_is_allocated;
2047
2048 if (required > remained)
2049 return -EPERM;
2050
2051 priv_dev->out_mem_is_allocated += required;
2052 priv_dev->onchip_used_size += required;
2053 }
2054
2055 return 0;
2056}
2057
2058static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
2059 struct cdns3_endpoint *priv_ep)
2060{
2061 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2062
2063 /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
2064 if (priv_dev->dev_ver <= DEV_VER_V2)
2065 writel(USB_CONF_DMULT, ®s->usb_conf);
2066
2067 if (priv_dev->dev_ver == DEV_VER_V2)
2068 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2);
2069
2070 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2071 u32 mask;
2072
2073 if (priv_ep->dir)
2074 mask = BIT(priv_ep->num + 16);
2075 else
2076 mask = BIT(priv_ep->num);
2077
2078 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
2079 cdns3_set_register_bit(®s->tdl_from_trb, mask);
2080 cdns3_set_register_bit(®s->tdl_beh, mask);
2081 cdns3_set_register_bit(®s->tdl_beh2, mask);
2082 cdns3_set_register_bit(®s->dma_adv_td, mask);
2083 }
2084
2085 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2086 cdns3_set_register_bit(®s->tdl_from_trb, mask);
2087
2088 cdns3_set_register_bit(®s->dtrans, mask);
2089 }
2090}
2091
2092/**
2093 * cdns3_ep_config - Configure hardware endpoint
2094 * @priv_ep: extended endpoint object
2095 * @enable: set EP_CFG_ENABLE bit in ep_cfg register.
2096 */
2097int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2098{
2099 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2100 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2101 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2102 u32 max_packet_size = priv_ep->wMaxPacketSize;
2103 u8 maxburst = priv_ep->bMaxBurst;
2104 u32 ep_cfg = 0;
2105 u8 buffering;
2106 int ret;
2107
2108 buffering = priv_dev->ep_buf_size - 1;
2109
2110 cdns3_configure_dmult(priv_dev, priv_ep);
2111
2112 switch (priv_ep->type) {
2113 case USB_ENDPOINT_XFER_INT:
2114 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
2115
2116 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2117 ep_cfg |= EP_CFG_TDL_CHK;
2118 break;
2119 case USB_ENDPOINT_XFER_BULK:
2120 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
2121
2122 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2123 ep_cfg |= EP_CFG_TDL_CHK;
2124 break;
2125 default:
2126 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2127 buffering = (priv_ep->bMaxBurst + 1) * (priv_ep->mult + 1) - 1;
2128 }
2129
2130 switch (priv_dev->gadget.speed) {
2131 case USB_SPEED_FULL:
2132 max_packet_size = is_iso_ep ? 1023 : 64;
2133 break;
2134 case USB_SPEED_HIGH:
2135 max_packet_size = is_iso_ep ? 1024 : 512;
2136 break;
2137 case USB_SPEED_SUPER:
2138 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
2139 max_packet_size = 1024;
2140 maxburst = priv_dev->ep_buf_size - 1;
2141 }
2142 break;
2143 default:
2144 /* all other speed are not supported */
2145 return -EINVAL;
2146 }
2147
2148 if (max_packet_size == 1024)
2149 priv_ep->trb_burst_size = 128;
2150 else if (max_packet_size >= 512)
2151 priv_ep->trb_burst_size = 64;
2152 else
2153 priv_ep->trb_burst_size = 16;
2154
2155 /*
2156 * In versions preceding DEV_VER_V2, for example, iMX8QM, there exit the bugs
2157 * in the DMA. These bugs occur when the trb_burst_size exceeds 16 and the
2158 * address is not aligned to 128 Bytes (which is a product of the 64-bit AXI
2159 * and AXI maximum burst length of 16 or 0xF+1, dma_axi_ctrl0[3:0]). This
2160 * results in data corruption when it crosses the 4K border. The corruption
2161 * specifically occurs from the position (4K - (address & 0x7F)) to 4K.
2162 *
2163 * So force trb_burst_size to 16 at such platform.
2164 */
2165 if (priv_dev->dev_ver < DEV_VER_V2)
2166 priv_ep->trb_burst_size = 16;
2167
2168 buffering = min_t(u8, buffering, EP_CFG_BUFFERING_MAX);
2169 maxburst = min_t(u8, maxburst, EP_CFG_MAXBURST_MAX);
2170
2171 /* onchip buffer is only allocated before configuration */
2172 if (!priv_dev->hw_configured_flag) {
2173 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2174 !!priv_ep->dir);
2175 if (ret) {
2176 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2177 return ret;
2178 }
2179 }
2180
2181 if (enable)
2182 ep_cfg |= EP_CFG_ENABLE;
2183
2184 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2185 if (priv_dev->dev_ver >= DEV_VER_V3) {
2186 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2187
2188 /*
2189 * Stream capable endpoints are handled by using ep_tdl
2190 * register. Other endpoints use TDL from TRB feature.
2191 */
2192 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,
2193 mask);
2194 }
2195
2196 /* Enable Stream Bit TDL chk and SID chk */
2197 ep_cfg |= EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK;
2198 }
2199
2200 ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2201 EP_CFG_MULT(priv_ep->mult) | /* must match EP setting */
2202 EP_CFG_BUFFERING(buffering) |
2203 EP_CFG_MAXBURST(maxburst);
2204
2205 cdns3_select_ep(priv_dev, bEndpointAddress);
2206 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2207 priv_ep->flags |= EP_CONFIGURED;
2208
2209 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2210 priv_ep->name, ep_cfg);
2211
2212 return 0;
2213}
2214
2215/* Find correct direction for HW endpoint according to description */
2216static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2217 struct cdns3_endpoint *priv_ep)
2218{
2219 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2220 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2221}
2222
2223static struct
2224cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2225 struct usb_endpoint_descriptor *desc)
2226{
2227 struct usb_ep *ep;
2228 struct cdns3_endpoint *priv_ep;
2229
2230 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2231 unsigned long num;
2232 int ret;
2233 /* ep name pattern likes epXin or epXout */
2234 char c[2] = {ep->name[2], '\0'};
2235
2236 ret = kstrtoul(c, 10, &num);
2237 if (ret)
2238 return ERR_PTR(ret);
2239
2240 priv_ep = ep_to_cdns3_ep(ep);
2241 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2242 if (!(priv_ep->flags & EP_CLAIMED)) {
2243 priv_ep->num = num;
2244 return priv_ep;
2245 }
2246 }
2247 }
2248
2249 return ERR_PTR(-ENOENT);
2250}
2251
2252/*
2253 * Cadence IP has one limitation that all endpoints must be configured
2254 * (Type & MaxPacketSize) before setting configuration through hardware
2255 * register, it means we can't change endpoints configuration after
2256 * set_configuration.
2257 *
2258 * This function set EP_CLAIMED flag which is added when the gadget driver
2259 * uses usb_ep_autoconfig to configure specific endpoint;
2260 * When the udc driver receives set_configurion request,
2261 * it goes through all claimed endpoints, and configure all endpoints
2262 * accordingly.
2263 *
2264 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2265 * ep_cfg register which can be changed after set_configuration, and do
2266 * some software operation accordingly.
2267 */
2268static struct
2269usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2270 struct usb_endpoint_descriptor *desc,
2271 struct usb_ss_ep_comp_descriptor *comp_desc)
2272{
2273 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2274 struct cdns3_endpoint *priv_ep;
2275 unsigned long flags;
2276
2277 priv_ep = cdns3_find_available_ep(priv_dev, desc);
2278 if (IS_ERR(priv_ep)) {
2279 dev_err(priv_dev->dev, "no available ep\n");
2280 return NULL;
2281 }
2282
2283 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2284
2285 spin_lock_irqsave(&priv_dev->lock, flags);
2286 priv_ep->endpoint.desc = desc;
2287 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2288 priv_ep->type = usb_endpoint_type(desc);
2289 priv_ep->flags |= EP_CLAIMED;
2290 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2291 priv_ep->wMaxPacketSize = usb_endpoint_maxp(desc);
2292 priv_ep->mult = USB_EP_MAXP_MULT(priv_ep->wMaxPacketSize);
2293 priv_ep->wMaxPacketSize &= USB_ENDPOINT_MAXP_MASK;
2294 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && comp_desc) {
2295 priv_ep->mult = USB_SS_MULT(comp_desc->bmAttributes) - 1;
2296 priv_ep->bMaxBurst = comp_desc->bMaxBurst;
2297 }
2298
2299 spin_unlock_irqrestore(&priv_dev->lock, flags);
2300 return &priv_ep->endpoint;
2301}
2302
2303/**
2304 * cdns3_gadget_ep_alloc_request - Allocates request
2305 * @ep: endpoint object associated with request
2306 * @gfp_flags: gfp flags
2307 *
2308 * Returns allocated request address, NULL on allocation error
2309 */
2310struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2311 gfp_t gfp_flags)
2312{
2313 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2314 struct cdns3_request *priv_req;
2315
2316 priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2317 if (!priv_req)
2318 return NULL;
2319
2320 priv_req->priv_ep = priv_ep;
2321
2322 trace_cdns3_alloc_request(priv_req);
2323 return &priv_req->request;
2324}
2325
2326/**
2327 * cdns3_gadget_ep_free_request - Free memory occupied by request
2328 * @ep: endpoint object associated with request
2329 * @request: request to free memory
2330 */
2331void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2332 struct usb_request *request)
2333{
2334 struct cdns3_request *priv_req = to_cdns3_request(request);
2335
2336 if (priv_req->aligned_buf)
2337 priv_req->aligned_buf->in_use = 0;
2338
2339 trace_cdns3_free_request(priv_req);
2340 kfree(priv_req);
2341}
2342
2343/**
2344 * cdns3_gadget_ep_enable - Enable endpoint
2345 * @ep: endpoint object
2346 * @desc: endpoint descriptor
2347 *
2348 * Returns 0 on success, error code elsewhere
2349 */
2350static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2351 const struct usb_endpoint_descriptor *desc)
2352{
2353 struct cdns3_endpoint *priv_ep;
2354 struct cdns3_device *priv_dev;
2355 const struct usb_ss_ep_comp_descriptor *comp_desc;
2356 u32 reg = EP_STS_EN_TRBERREN;
2357 u32 bEndpointAddress;
2358 unsigned long flags;
2359 int enable = 1;
2360 int ret = 0;
2361 int val;
2362
2363 if (!ep) {
2364 pr_debug("usbss: ep not configured?\n");
2365 return -EINVAL;
2366 }
2367
2368 priv_ep = ep_to_cdns3_ep(ep);
2369 priv_dev = priv_ep->cdns3_dev;
2370 comp_desc = priv_ep->endpoint.comp_desc;
2371
2372 if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2373 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2374 return -EINVAL;
2375 }
2376
2377 if (!desc->wMaxPacketSize) {
2378 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2379 return -EINVAL;
2380 }
2381
2382 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2383 "%s is already enabled\n", priv_ep->name))
2384 return 0;
2385
2386 spin_lock_irqsave(&priv_dev->lock, flags);
2387
2388 priv_ep->endpoint.desc = desc;
2389 priv_ep->type = usb_endpoint_type(desc);
2390 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2391
2392 if (priv_ep->interval > ISO_MAX_INTERVAL &&
2393 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2394 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2395 ISO_MAX_INTERVAL);
2396
2397 ret = -EINVAL;
2398 goto exit;
2399 }
2400
2401 bEndpointAddress = priv_ep->num | priv_ep->dir;
2402 cdns3_select_ep(priv_dev, bEndpointAddress);
2403
2404 /*
2405 * For some versions of controller at some point during ISO OUT traffic
2406 * DMA reads Transfer Ring for the EP which has never got doorbell.
2407 * This issue was detected only on simulation, but to avoid this issue
2408 * driver add protection against it. To fix it driver enable ISO OUT
2409 * endpoint before setting DRBL. This special treatment of ISO OUT
2410 * endpoints are recommended by controller specification.
2411 */
2412 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2413 enable = 0;
2414
2415 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2416 /*
2417 * Enable stream support (SS mode) related interrupts
2418 * in EP_STS_EN Register
2419 */
2420 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2421 reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2422 EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2423 EP_STS_EN_STREAMREN;
2424 priv_ep->use_streams = true;
2425 ret = cdns3_ep_config(priv_ep, enable);
2426 priv_dev->using_streams |= true;
2427 }
2428 } else {
2429 ret = cdns3_ep_config(priv_ep, enable);
2430 }
2431
2432 if (ret)
2433 goto exit;
2434
2435 ret = cdns3_allocate_trb_pool(priv_ep);
2436 if (ret)
2437 goto exit;
2438
2439 bEndpointAddress = priv_ep->num | priv_ep->dir;
2440 cdns3_select_ep(priv_dev, bEndpointAddress);
2441
2442 trace_cdns3_gadget_ep_enable(priv_ep);
2443
2444 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2445
2446 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2447 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2448 1, 1000);
2449
2450 if (unlikely(ret)) {
2451 cdns3_free_trb_pool(priv_ep);
2452 ret = -EINVAL;
2453 goto exit;
2454 }
2455
2456 /* enable interrupt for selected endpoint */
2457 cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2458 BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2459
2460 if (priv_dev->dev_ver < DEV_VER_V2)
2461 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2462
2463 writel(reg, &priv_dev->regs->ep_sts_en);
2464
2465 ep->desc = desc;
2466 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2467 EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2468 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2469 priv_ep->wa1_set = 0;
2470 priv_ep->enqueue = 0;
2471 priv_ep->dequeue = 0;
2472 reg = readl(&priv_dev->regs->ep_sts);
2473 priv_ep->pcs = !!EP_STS_CCS(reg);
2474 priv_ep->ccs = !!EP_STS_CCS(reg);
2475 /* one TRB is reserved for link TRB used in DMULT mode*/
2476 priv_ep->free_trbs = priv_ep->num_trbs - 1;
2477exit:
2478 spin_unlock_irqrestore(&priv_dev->lock, flags);
2479
2480 return ret;
2481}
2482
2483/**
2484 * cdns3_gadget_ep_disable - Disable endpoint
2485 * @ep: endpoint object
2486 *
2487 * Returns 0 on success, error code elsewhere
2488 */
2489static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2490{
2491 struct cdns3_endpoint *priv_ep;
2492 struct cdns3_request *priv_req;
2493 struct cdns3_device *priv_dev;
2494 struct usb_request *request;
2495 unsigned long flags;
2496 int ret = 0;
2497 u32 ep_cfg;
2498 int val;
2499
2500 if (!ep) {
2501 pr_err("usbss: invalid parameters\n");
2502 return -EINVAL;
2503 }
2504
2505 priv_ep = ep_to_cdns3_ep(ep);
2506 priv_dev = priv_ep->cdns3_dev;
2507
2508 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2509 "%s is already disabled\n", priv_ep->name))
2510 return 0;
2511
2512 spin_lock_irqsave(&priv_dev->lock, flags);
2513
2514 trace_cdns3_gadget_ep_disable(priv_ep);
2515
2516 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2517
2518 ep_cfg = readl(&priv_dev->regs->ep_cfg);
2519 ep_cfg &= ~EP_CFG_ENABLE;
2520 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2521
2522 /**
2523 * Driver needs some time before resetting endpoint.
2524 * It need waits for clearing DBUSY bit or for timeout expired.
2525 * 10us is enough time for controller to stop transfer.
2526 */
2527 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2528 !(val & EP_STS_DBUSY), 1, 10);
2529 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2530
2531 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2532 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2533 1, 1000);
2534 if (unlikely(ret))
2535 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2536 priv_ep->name);
2537
2538 while (!list_empty(&priv_ep->pending_req_list)) {
2539 request = cdns3_next_request(&priv_ep->pending_req_list);
2540
2541 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2542 -ESHUTDOWN);
2543 }
2544
2545 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2546 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2547 list_del_init(&priv_req->list);
2548
2549 kfree(priv_req->request.buf);
2550 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2551 &priv_req->request);
2552 --priv_ep->wa2_counter;
2553 }
2554
2555 while (!list_empty(&priv_ep->deferred_req_list)) {
2556 request = cdns3_next_request(&priv_ep->deferred_req_list);
2557
2558 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2559 -ESHUTDOWN);
2560 }
2561
2562 priv_ep->descmis_req = NULL;
2563
2564 ep->desc = NULL;
2565 priv_ep->flags &= ~EP_ENABLED;
2566 priv_ep->use_streams = false;
2567
2568 spin_unlock_irqrestore(&priv_dev->lock, flags);
2569
2570 return ret;
2571}
2572
2573/**
2574 * __cdns3_gadget_ep_queue - Transfer data on endpoint
2575 * @ep: endpoint object
2576 * @request: request object
2577 * @gfp_flags: gfp flags
2578 *
2579 * Returns 0 on success, error code elsewhere
2580 */
2581static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2582 struct usb_request *request,
2583 gfp_t gfp_flags)
2584{
2585 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2586 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2587 struct cdns3_request *priv_req;
2588 int ret = 0;
2589
2590 request->actual = 0;
2591 request->status = -EINPROGRESS;
2592 priv_req = to_cdns3_request(request);
2593 trace_cdns3_ep_queue(priv_req);
2594
2595 if (priv_dev->dev_ver < DEV_VER_V2) {
2596 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2597 priv_req);
2598
2599 if (ret == EINPROGRESS)
2600 return 0;
2601 }
2602
2603 ret = cdns3_prepare_aligned_request_buf(priv_req);
2604 if (ret < 0)
2605 return ret;
2606
2607 if (likely(!(priv_req->flags & REQUEST_UNALIGNED))) {
2608 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2609 usb_endpoint_dir_in(ep->desc));
2610 if (ret)
2611 return ret;
2612 }
2613
2614 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2615
2616 /*
2617 * For stream capable endpoint if prime irq flag is set then only start
2618 * request.
2619 * If hardware endpoint configuration has not been set yet then
2620 * just queue request in deferred list. Transfer will be started in
2621 * cdns3_set_hw_configuration.
2622 */
2623 if (!request->stream_id) {
2624 if (priv_dev->hw_configured_flag &&
2625 !(priv_ep->flags & EP_STALLED) &&
2626 !(priv_ep->flags & EP_STALL_PENDING))
2627 cdns3_start_all_request(priv_dev, priv_ep);
2628 } else {
2629 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2630 cdns3_start_all_request(priv_dev, priv_ep);
2631 }
2632
2633 return 0;
2634}
2635
2636static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2637 gfp_t gfp_flags)
2638{
2639 struct usb_request *zlp_request;
2640 struct cdns3_endpoint *priv_ep;
2641 struct cdns3_device *priv_dev;
2642 unsigned long flags;
2643 int ret;
2644
2645 if (!request || !ep)
2646 return -EINVAL;
2647
2648 priv_ep = ep_to_cdns3_ep(ep);
2649 priv_dev = priv_ep->cdns3_dev;
2650
2651 spin_lock_irqsave(&priv_dev->lock, flags);
2652
2653 ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2654
2655 if (ret == 0 && request->zero && request->length &&
2656 (request->length % ep->maxpacket == 0)) {
2657 struct cdns3_request *priv_req;
2658
2659 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2660 zlp_request->buf = priv_dev->zlp_buf;
2661 zlp_request->length = 0;
2662
2663 priv_req = to_cdns3_request(zlp_request);
2664 priv_req->flags |= REQUEST_ZLP;
2665
2666 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2667 priv_ep->name);
2668 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2669 }
2670
2671 spin_unlock_irqrestore(&priv_dev->lock, flags);
2672 return ret;
2673}
2674
2675/**
2676 * cdns3_gadget_ep_dequeue - Remove request from transfer queue
2677 * @ep: endpoint object associated with request
2678 * @request: request object
2679 *
2680 * Returns 0 on success, error code elsewhere
2681 */
2682int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2683 struct usb_request *request)
2684{
2685 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2686 struct cdns3_device *priv_dev;
2687 struct usb_request *req, *req_temp;
2688 struct cdns3_request *priv_req;
2689 struct cdns3_trb *link_trb;
2690 u8 req_on_hw_ring = 0;
2691 unsigned long flags;
2692 int ret = 0;
2693 int val;
2694
2695 if (!ep || !request || !ep->desc)
2696 return -EINVAL;
2697
2698 priv_dev = priv_ep->cdns3_dev;
2699
2700 spin_lock_irqsave(&priv_dev->lock, flags);
2701
2702 priv_req = to_cdns3_request(request);
2703
2704 trace_cdns3_ep_dequeue(priv_req);
2705
2706 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2707
2708 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2709 list) {
2710 if (request == req) {
2711 req_on_hw_ring = 1;
2712 goto found;
2713 }
2714 }
2715
2716 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2717 list) {
2718 if (request == req)
2719 goto found;
2720 }
2721
2722 goto not_found;
2723
2724found:
2725 link_trb = priv_req->trb;
2726
2727 /* Update ring only if removed request is on pending_req_list list */
2728 if (req_on_hw_ring && link_trb) {
2729 /* Stop DMA */
2730 writel(EP_CMD_DFLUSH, &priv_dev->regs->ep_cmd);
2731
2732 /* wait for DFLUSH cleared */
2733 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2734 !(val & EP_CMD_DFLUSH), 1, 1000);
2735
2736 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2737 ((priv_req->end_trb + 1) * TRB_SIZE)));
2738 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
2739 TRB_TYPE(TRB_LINK) | TRB_CHAIN);
2740
2741 if (priv_ep->wa1_trb == priv_req->trb)
2742 cdns3_wa1_restore_cycle_bit(priv_ep);
2743 }
2744
2745 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2746
2747 req = cdns3_next_request(&priv_ep->pending_req_list);
2748 if (req)
2749 cdns3_rearm_transfer(priv_ep, 1);
2750
2751not_found:
2752 spin_unlock_irqrestore(&priv_dev->lock, flags);
2753 return ret;
2754}
2755
2756/**
2757 * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint
2758 * Should be called after acquiring spin_lock and selecting ep
2759 * @priv_ep: endpoint object to set stall on.
2760 */
2761void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2762{
2763 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2764
2765 trace_cdns3_halt(priv_ep, 1, 0);
2766
2767 if (!(priv_ep->flags & EP_STALLED)) {
2768 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2769
2770 if (!(ep_sts_reg & EP_STS_DBUSY))
2771 cdns3_ep_stall_flush(priv_ep);
2772 else
2773 priv_ep->flags |= EP_STALL_PENDING;
2774 }
2775}
2776
2777/**
2778 * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint
2779 * Should be called after acquiring spin_lock and selecting ep
2780 * @priv_ep: endpoint object to clear stall on
2781 */
2782int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2783{
2784 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2785 struct usb_request *request;
2786 struct cdns3_request *priv_req;
2787 struct cdns3_trb *trb = NULL;
2788 struct cdns3_trb trb_tmp;
2789 int ret;
2790 int val;
2791
2792 trace_cdns3_halt(priv_ep, 0, 0);
2793
2794 request = cdns3_next_request(&priv_ep->pending_req_list);
2795 if (request) {
2796 priv_req = to_cdns3_request(request);
2797 trb = priv_req->trb;
2798 if (trb) {
2799 trb_tmp = *trb;
2800 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2801 }
2802 }
2803
2804 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2805
2806 /* wait for EPRST cleared */
2807 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2808 !(val & EP_CMD_EPRST), 1, 100);
2809 if (ret)
2810 return -EINVAL;
2811
2812 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2813
2814 if (request) {
2815 if (trb)
2816 *trb = trb_tmp;
2817
2818 cdns3_rearm_transfer(priv_ep, 1);
2819 }
2820
2821 cdns3_start_all_request(priv_dev, priv_ep);
2822 return ret;
2823}
2824
2825/**
2826 * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint
2827 * @ep: endpoint object to set/clear stall on
2828 * @value: 1 for set stall, 0 for clear stall
2829 *
2830 * Returns 0 on success, error code elsewhere
2831 */
2832int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2833{
2834 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2835 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2836 unsigned long flags;
2837 int ret = 0;
2838
2839 if (!(priv_ep->flags & EP_ENABLED))
2840 return -EPERM;
2841
2842 spin_lock_irqsave(&priv_dev->lock, flags);
2843
2844 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2845
2846 if (!value) {
2847 priv_ep->flags &= ~EP_WEDGE;
2848 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2849 } else {
2850 __cdns3_gadget_ep_set_halt(priv_ep);
2851 }
2852
2853 spin_unlock_irqrestore(&priv_dev->lock, flags);
2854
2855 return ret;
2856}
2857
2858extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2859
2860static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2861 .enable = cdns3_gadget_ep_enable,
2862 .disable = cdns3_gadget_ep_disable,
2863 .alloc_request = cdns3_gadget_ep_alloc_request,
2864 .free_request = cdns3_gadget_ep_free_request,
2865 .queue = cdns3_gadget_ep_queue,
2866 .dequeue = cdns3_gadget_ep_dequeue,
2867 .set_halt = cdns3_gadget_ep_set_halt,
2868 .set_wedge = cdns3_gadget_ep_set_wedge,
2869};
2870
2871/**
2872 * cdns3_gadget_get_frame - Returns number of actual ITP frame
2873 * @gadget: gadget object
2874 *
2875 * Returns number of actual ITP frame
2876 */
2877static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2878{
2879 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2880
2881 return readl(&priv_dev->regs->usb_itpn);
2882}
2883
2884int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2885{
2886 enum usb_device_speed speed;
2887
2888 speed = cdns3_get_speed(priv_dev);
2889
2890 if (speed >= USB_SPEED_SUPER)
2891 return 0;
2892
2893 /* Start driving resume signaling to indicate remote wakeup. */
2894 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2895
2896 return 0;
2897}
2898
2899static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2900{
2901 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2902 unsigned long flags;
2903 int ret = 0;
2904
2905 spin_lock_irqsave(&priv_dev->lock, flags);
2906 ret = __cdns3_gadget_wakeup(priv_dev);
2907 spin_unlock_irqrestore(&priv_dev->lock, flags);
2908 return ret;
2909}
2910
2911static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2912 int is_selfpowered)
2913{
2914 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2915 unsigned long flags;
2916
2917 spin_lock_irqsave(&priv_dev->lock, flags);
2918 priv_dev->is_selfpowered = !!is_selfpowered;
2919 spin_unlock_irqrestore(&priv_dev->lock, flags);
2920 return 0;
2921}
2922
2923static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2924{
2925 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2926
2927 if (is_on) {
2928 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2929 } else {
2930 writel(~0, &priv_dev->regs->ep_ists);
2931 writel(~0, &priv_dev->regs->usb_ists);
2932 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2933 }
2934
2935 return 0;
2936}
2937
2938static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2939{
2940 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2941 u32 reg;
2942
2943 cdns3_ep0_config(priv_dev);
2944
2945 /* enable interrupts for endpoint 0 (in and out) */
2946 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien);
2947
2948 /*
2949 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2950 * revision of controller.
2951 */
2952 if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2953 reg = readl(®s->dbg_link1);
2954
2955 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2956 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2957 DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2958 writel(reg, ®s->dbg_link1);
2959 }
2960
2961 /*
2962 * By default some platforms has set protected access to memory.
2963 * This cause problem with cache, so driver restore non-secure
2964 * access to memory.
2965 */
2966 reg = readl(®s->dma_axi_ctrl);
2967 reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2968 DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2969 writel(reg, ®s->dma_axi_ctrl);
2970
2971 /* enable generic interrupt*/
2972 writel(USB_IEN_INIT, ®s->usb_ien);
2973 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf);
2974 /* keep Fast Access bit */
2975 writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);
2976
2977 cdns3_configure_dmult(priv_dev, NULL);
2978}
2979
2980/**
2981 * cdns3_gadget_udc_start - Gadget start
2982 * @gadget: gadget object
2983 * @driver: driver which operates on this gadget
2984 *
2985 * Returns 0 on success, error code elsewhere
2986 */
2987static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2988 struct usb_gadget_driver *driver)
2989{
2990 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2991 unsigned long flags;
2992 enum usb_device_speed max_speed = driver->max_speed;
2993
2994 spin_lock_irqsave(&priv_dev->lock, flags);
2995 priv_dev->gadget_driver = driver;
2996
2997 /* limit speed if necessary */
2998 max_speed = min(driver->max_speed, gadget->max_speed);
2999
3000 switch (max_speed) {
3001 case USB_SPEED_FULL:
3002 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
3003 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
3004 break;
3005 case USB_SPEED_HIGH:
3006 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
3007 break;
3008 case USB_SPEED_SUPER:
3009 break;
3010 default:
3011 dev_err(priv_dev->dev,
3012 "invalid maximum_speed parameter %d\n",
3013 max_speed);
3014 fallthrough;
3015 case USB_SPEED_UNKNOWN:
3016 /* default to superspeed */
3017 max_speed = USB_SPEED_SUPER;
3018 break;
3019 }
3020
3021 cdns3_gadget_config(priv_dev);
3022 spin_unlock_irqrestore(&priv_dev->lock, flags);
3023 return 0;
3024}
3025
3026/**
3027 * cdns3_gadget_udc_stop - Stops gadget
3028 * @gadget: gadget object
3029 *
3030 * Returns 0
3031 */
3032static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
3033{
3034 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
3035 struct cdns3_endpoint *priv_ep;
3036 u32 bEndpointAddress;
3037 struct usb_ep *ep;
3038 int val;
3039
3040 priv_dev->gadget_driver = NULL;
3041
3042 priv_dev->onchip_used_size = 0;
3043 priv_dev->out_mem_is_allocated = 0;
3044 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3045
3046 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
3047 priv_ep = ep_to_cdns3_ep(ep);
3048 bEndpointAddress = priv_ep->num | priv_ep->dir;
3049 cdns3_select_ep(priv_dev, bEndpointAddress);
3050 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
3051 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
3052 !(val & EP_CMD_EPRST), 1, 100);
3053
3054 priv_ep->flags &= ~EP_CLAIMED;
3055 }
3056
3057 /* disable interrupt for device */
3058 writel(0, &priv_dev->regs->usb_ien);
3059 writel(0, &priv_dev->regs->usb_pwr);
3060 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
3061
3062 return 0;
3063}
3064
3065/**
3066 * cdns3_gadget_check_config - ensure cdns3 can support the USB configuration
3067 * @gadget: pointer to the USB gadget
3068 *
3069 * Used to record the maximum number of endpoints being used in a USB composite
3070 * device. (across all configurations) This is to be used in the calculation
3071 * of the TXFIFO sizes when resizing internal memory for individual endpoints.
3072 * It will help ensured that the resizing logic reserves enough space for at
3073 * least one max packet.
3074 */
3075static int cdns3_gadget_check_config(struct usb_gadget *gadget)
3076{
3077 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
3078 struct cdns3_endpoint *priv_ep;
3079 struct usb_ep *ep;
3080 int n_in = 0;
3081 int iso = 0;
3082 int out = 1;
3083 int total;
3084 int n;
3085
3086 list_for_each_entry(ep, &gadget->ep_list, ep_list) {
3087 priv_ep = ep_to_cdns3_ep(ep);
3088 if (!(priv_ep->flags & EP_CLAIMED))
3089 continue;
3090
3091 n = (priv_ep->mult + 1) * (priv_ep->bMaxBurst + 1);
3092 if (ep->address & USB_DIR_IN) {
3093 /*
3094 * ISO transfer: DMA start move data when get ISO, only transfer
3095 * data as min(TD size, iso). No benefit for allocate bigger
3096 * internal memory than 'iso'.
3097 */
3098 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
3099 iso += n;
3100 else
3101 n_in++;
3102 } else {
3103 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
3104 out = max_t(int, out, n);
3105 }
3106 }
3107
3108 /* 2KB are reserved for EP0, 1KB for out*/
3109 total = 2 + n_in + out + iso;
3110
3111 if (total > priv_dev->onchip_buffers)
3112 return -ENOMEM;
3113
3114 priv_dev->ep_buf_size = (priv_dev->onchip_buffers - 2 - iso) / (n_in + out);
3115
3116 return 0;
3117}
3118
3119static const struct usb_gadget_ops cdns3_gadget_ops = {
3120 .get_frame = cdns3_gadget_get_frame,
3121 .wakeup = cdns3_gadget_wakeup,
3122 .set_selfpowered = cdns3_gadget_set_selfpowered,
3123 .pullup = cdns3_gadget_pullup,
3124 .udc_start = cdns3_gadget_udc_start,
3125 .udc_stop = cdns3_gadget_udc_stop,
3126 .match_ep = cdns3_gadget_match_ep,
3127 .check_config = cdns3_gadget_check_config,
3128};
3129
3130static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
3131{
3132 int i;
3133
3134 /* ep0 OUT point to ep0 IN. */
3135 priv_dev->eps[16] = NULL;
3136
3137 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
3138 if (priv_dev->eps[i]) {
3139 cdns3_free_trb_pool(priv_dev->eps[i]);
3140 devm_kfree(priv_dev->dev, priv_dev->eps[i]);
3141 }
3142}
3143
3144/**
3145 * cdns3_init_eps - Initializes software endpoints of gadget
3146 * @priv_dev: extended gadget object
3147 *
3148 * Returns 0 on success, error code elsewhere
3149 */
3150static int cdns3_init_eps(struct cdns3_device *priv_dev)
3151{
3152 u32 ep_enabled_reg, iso_ep_reg;
3153 struct cdns3_endpoint *priv_ep;
3154 int ep_dir, ep_number;
3155 u32 ep_mask;
3156 int ret = 0;
3157 int i;
3158
3159 /* Read it from USB_CAP3 to USB_CAP5 */
3160 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
3161 iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
3162
3163 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
3164
3165 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
3166 ep_dir = i >> 4; /* i div 16 */
3167 ep_number = i & 0xF; /* i % 16 */
3168 ep_mask = BIT(i);
3169
3170 if (!(ep_enabled_reg & ep_mask))
3171 continue;
3172
3173 if (ep_dir && !ep_number) {
3174 priv_dev->eps[i] = priv_dev->eps[0];
3175 continue;
3176 }
3177
3178 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3179 GFP_KERNEL);
3180 if (!priv_ep)
3181 goto err;
3182
3183 /* set parent of endpoint object */
3184 priv_ep->cdns3_dev = priv_dev;
3185 priv_dev->eps[i] = priv_ep;
3186 priv_ep->num = ep_number;
3187 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3188
3189 if (!ep_number) {
3190 ret = cdns3_init_ep0(priv_dev, priv_ep);
3191 if (ret) {
3192 dev_err(priv_dev->dev, "Failed to init ep0\n");
3193 goto err;
3194 }
3195 } else {
3196 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3197 ep_number, !!ep_dir ? "in" : "out");
3198 priv_ep->endpoint.name = priv_ep->name;
3199
3200 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3201 CDNS3_EP_MAX_PACKET_LIMIT);
3202 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3203 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3204 if (ep_dir)
3205 priv_ep->endpoint.caps.dir_in = 1;
3206 else
3207 priv_ep->endpoint.caps.dir_out = 1;
3208
3209 if (iso_ep_reg & ep_mask)
3210 priv_ep->endpoint.caps.type_iso = 1;
3211
3212 priv_ep->endpoint.caps.type_bulk = 1;
3213 priv_ep->endpoint.caps.type_int = 1;
3214
3215 list_add_tail(&priv_ep->endpoint.ep_list,
3216 &priv_dev->gadget.ep_list);
3217 }
3218
3219 priv_ep->flags = 0;
3220
3221 dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n",
3222 priv_ep->name,
3223 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3224 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3225
3226 INIT_LIST_HEAD(&priv_ep->pending_req_list);
3227 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3228 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
3229 }
3230
3231 return 0;
3232err:
3233 cdns3_free_all_eps(priv_dev);
3234 return -ENOMEM;
3235}
3236
3237static void cdns3_gadget_release(struct device *dev)
3238{
3239 struct cdns3_device *priv_dev = container_of(dev,
3240 struct cdns3_device, gadget.dev);
3241
3242 kfree(priv_dev);
3243}
3244
3245static void cdns3_gadget_exit(struct cdns *cdns)
3246{
3247 struct cdns3_device *priv_dev;
3248
3249 priv_dev = cdns->gadget_dev;
3250
3251
3252 pm_runtime_mark_last_busy(cdns->dev);
3253 pm_runtime_put_autosuspend(cdns->dev);
3254
3255 usb_del_gadget(&priv_dev->gadget);
3256 devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
3257
3258 cdns3_free_all_eps(priv_dev);
3259
3260 while (!list_empty(&priv_dev->aligned_buf_list)) {
3261 struct cdns3_aligned_buf *buf;
3262
3263 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3264 dma_free_noncoherent(priv_dev->sysdev, buf->size,
3265 buf->buf,
3266 buf->dma,
3267 buf->dir);
3268
3269 list_del(&buf->list);
3270 kfree(buf);
3271 }
3272
3273 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3274 priv_dev->setup_dma);
3275 dma_pool_destroy(priv_dev->eps_dma_pool);
3276
3277 kfree(priv_dev->zlp_buf);
3278 usb_put_gadget(&priv_dev->gadget);
3279 cdns->gadget_dev = NULL;
3280 cdns_drd_gadget_off(cdns);
3281}
3282
3283static int cdns3_gadget_start(struct cdns *cdns)
3284{
3285 struct cdns3_device *priv_dev;
3286 u32 max_speed;
3287 int ret;
3288
3289 priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3290 if (!priv_dev)
3291 return -ENOMEM;
3292
3293 usb_initialize_gadget(cdns->dev, &priv_dev->gadget,
3294 cdns3_gadget_release);
3295 cdns->gadget_dev = priv_dev;
3296 priv_dev->sysdev = cdns->dev;
3297 priv_dev->dev = cdns->dev;
3298 priv_dev->regs = cdns->dev_regs;
3299
3300 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3301 &priv_dev->onchip_buffers);
3302
3303 if (priv_dev->onchip_buffers <= 0) {
3304 u32 reg = readl(&priv_dev->regs->usb_cap2);
3305
3306 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3307 }
3308
3309 if (!priv_dev->onchip_buffers)
3310 priv_dev->onchip_buffers = 256;
3311
3312 max_speed = usb_get_maximum_speed(cdns->dev);
3313
3314 /* Check the maximum_speed parameter */
3315 switch (max_speed) {
3316 case USB_SPEED_FULL:
3317 case USB_SPEED_HIGH:
3318 case USB_SPEED_SUPER:
3319 break;
3320 default:
3321 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3322 max_speed);
3323 fallthrough;
3324 case USB_SPEED_UNKNOWN:
3325 /* default to superspeed */
3326 max_speed = USB_SPEED_SUPER;
3327 break;
3328 }
3329
3330 /* fill gadget fields */
3331 priv_dev->gadget.max_speed = max_speed;
3332 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3333 priv_dev->gadget.ops = &cdns3_gadget_ops;
3334 priv_dev->gadget.name = "usb-ss-gadget";
3335 priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3336 priv_dev->gadget.irq = cdns->dev_irq;
3337
3338 spin_lock_init(&priv_dev->lock);
3339 INIT_WORK(&priv_dev->pending_status_wq,
3340 cdns3_pending_setup_status_handler);
3341
3342 INIT_WORK(&priv_dev->aligned_buf_wq,
3343 cdns3_free_aligned_request_buf);
3344
3345 /* initialize endpoint container */
3346 INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3347 INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3348 priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool",
3349 priv_dev->sysdev,
3350 TRB_RING_SIZE, 8, 0);
3351 if (!priv_dev->eps_dma_pool) {
3352 dev_err(priv_dev->dev, "Failed to create TRB dma pool\n");
3353 ret = -ENOMEM;
3354 goto err1;
3355 }
3356
3357 ret = cdns3_init_eps(priv_dev);
3358 if (ret) {
3359 dev_err(priv_dev->dev, "Failed to create endpoints\n");
3360 goto err1;
3361 }
3362
3363 /* allocate memory for setup packet buffer */
3364 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3365 &priv_dev->setup_dma, GFP_DMA);
3366 if (!priv_dev->setup_buf) {
3367 ret = -ENOMEM;
3368 goto err2;
3369 }
3370
3371 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3372
3373 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3374 readl(&priv_dev->regs->usb_cap6));
3375 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3376 readl(&priv_dev->regs->usb_cap1));
3377 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3378 readl(&priv_dev->regs->usb_cap2));
3379
3380 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3381 if (priv_dev->dev_ver >= DEV_VER_V2)
3382 priv_dev->gadget.sg_supported = 1;
3383
3384 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3385 if (!priv_dev->zlp_buf) {
3386 ret = -ENOMEM;
3387 goto err3;
3388 }
3389
3390 /* add USB gadget device */
3391 ret = usb_add_gadget(&priv_dev->gadget);
3392 if (ret < 0) {
3393 dev_err(priv_dev->dev, "Failed to add gadget\n");
3394 goto err4;
3395 }
3396
3397 return 0;
3398err4:
3399 kfree(priv_dev->zlp_buf);
3400err3:
3401 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3402 priv_dev->setup_dma);
3403err2:
3404 cdns3_free_all_eps(priv_dev);
3405err1:
3406 dma_pool_destroy(priv_dev->eps_dma_pool);
3407
3408 usb_put_gadget(&priv_dev->gadget);
3409 cdns->gadget_dev = NULL;
3410 return ret;
3411}
3412
3413static int __cdns3_gadget_init(struct cdns *cdns)
3414{
3415 int ret = 0;
3416
3417 /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3418 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3419 if (ret) {
3420 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3421 return ret;
3422 }
3423
3424 cdns_drd_gadget_on(cdns);
3425 pm_runtime_get_sync(cdns->dev);
3426
3427 ret = cdns3_gadget_start(cdns);
3428 if (ret) {
3429 pm_runtime_put_sync(cdns->dev);
3430 return ret;
3431 }
3432
3433 /*
3434 * Because interrupt line can be shared with other components in
3435 * driver it can't use IRQF_ONESHOT flag here.
3436 */
3437 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3438 cdns3_device_irq_handler,
3439 cdns3_device_thread_irq_handler,
3440 IRQF_SHARED, dev_name(cdns->dev),
3441 cdns->gadget_dev);
3442
3443 if (ret)
3444 goto err0;
3445
3446 return 0;
3447err0:
3448 cdns3_gadget_exit(cdns);
3449 return ret;
3450}
3451
3452static int cdns3_gadget_suspend(struct cdns *cdns, bool do_wakeup)
3453__must_hold(&cdns->lock)
3454{
3455 struct cdns3_device *priv_dev = cdns->gadget_dev;
3456
3457 spin_unlock(&cdns->lock);
3458 cdns3_disconnect_gadget(priv_dev);
3459 spin_lock(&cdns->lock);
3460
3461 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3462 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3463 cdns3_hw_reset_eps_config(priv_dev);
3464
3465 /* disable interrupt for device */
3466 writel(0, &priv_dev->regs->usb_ien);
3467
3468 return 0;
3469}
3470
3471static int cdns3_gadget_resume(struct cdns *cdns, bool hibernated)
3472{
3473 struct cdns3_device *priv_dev = cdns->gadget_dev;
3474
3475 if (!priv_dev->gadget_driver)
3476 return 0;
3477
3478 cdns3_gadget_config(priv_dev);
3479 if (hibernated)
3480 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
3481
3482 return 0;
3483}
3484
3485/**
3486 * cdns3_gadget_init - initialize device structure
3487 *
3488 * @cdns: cdns instance
3489 *
3490 * This function initializes the gadget.
3491 */
3492int cdns3_gadget_init(struct cdns *cdns)
3493{
3494 struct cdns_role_driver *rdrv;
3495
3496 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3497 if (!rdrv)
3498 return -ENOMEM;
3499
3500 rdrv->start = __cdns3_gadget_init;
3501 rdrv->stop = cdns3_gadget_exit;
3502 rdrv->suspend = cdns3_gadget_suspend;
3503 rdrv->resume = cdns3_gadget_resume;
3504 rdrv->state = CDNS_ROLE_STATE_INACTIVE;
3505 rdrv->name = "gadget";
3506 cdns->roles[USB_ROLE_DEVICE] = rdrv;
3507
3508 return 0;
3509}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Cadence USBSS DRD Driver - gadget side.
4 *
5 * Copyright (C) 2018-2019 Cadence Design Systems.
6 * Copyright (C) 2017-2018 NXP
7 *
8 * Authors: Pawel Jez <pjez@cadence.com>,
9 * Pawel Laszczak <pawell@cadence.com>
10 * Peter Chen <peter.chen@nxp.com>
11 */
12
13/*
14 * Work around 1:
15 * At some situations, the controller may get stale data address in TRB
16 * at below sequences:
17 * 1. Controller read TRB includes data address
18 * 2. Software updates TRBs includes data address and Cycle bit
19 * 3. Controller read TRB which includes Cycle bit
20 * 4. DMA run with stale data address
21 *
22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
23 * After preparing all TRBs driver needs to check the position of DMA and
24 * if the DMA point to the first just added TRB and doorbell is 1,
25 * then driver must defer making this TRB as valid. This TRB will be make
26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
27 * interrupt.
28 *
29 * Issue has been fixed in DEV_VER_V3 version of controller.
30 *
31 * Work around 2:
32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
34 * in correct order. If the first packet in the buffer will not be handled,
35 * then the following packets directed for other endpoints and functions
36 * will be blocked.
37 * Additionally the packets directed to one endpoint can block entire on-chip
38 * buffers. In this case transfer to other endpoints also will blocked.
39 *
40 * To resolve this issue after raising the descriptor missing interrupt
41 * driver prepares internal usb_request object and use it to arm DMA transfer.
42 *
43 * The problematic situation was observed in case when endpoint has been enabled
44 * but no usb_request were queued. Driver try detects such endpoints and will
45 * use this workaround only for these endpoint.
46 *
47 * Driver use limited number of buffer. This number can be set by macro
48 * CDNS3_WA2_NUM_BUFFERS.
49 *
50 * Such blocking situation was observed on ACM gadget. For this function
51 * host send OUT data packet but ACM function is not prepared for this packet.
52 * It's cause that buffer placed in on chip memory block transfer to other
53 * endpoints.
54 *
55 * Issue has been fixed in DEV_VER_V2 version of controller.
56 *
57 */
58
59#include <linux/dma-mapping.h>
60#include <linux/usb/gadget.h>
61#include <linux/module.h>
62#include <linux/dmapool.h>
63#include <linux/iopoll.h>
64
65#include "core.h"
66#include "gadget-export.h"
67#include "cdns3-gadget.h"
68#include "cdns3-trace.h"
69#include "drd.h"
70
71static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
72 struct usb_request *request,
73 gfp_t gfp_flags);
74
75static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
76 struct usb_request *request);
77
78static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
79 struct usb_request *request);
80
81/**
82 * cdns3_clear_register_bit - clear bit in given register.
83 * @ptr: address of device controller register to be read and changed
84 * @mask: bits requested to clar
85 */
86static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
87{
88 mask = readl(ptr) & ~mask;
89 writel(mask, ptr);
90}
91
92/**
93 * cdns3_set_register_bit - set bit in given register.
94 * @ptr: address of device controller register to be read and changed
95 * @mask: bits requested to set
96 */
97void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
98{
99 mask = readl(ptr) | mask;
100 writel(mask, ptr);
101}
102
103/**
104 * cdns3_ep_addr_to_index - Macro converts endpoint address to
105 * index of endpoint object in cdns3_device.eps[] container
106 * @ep_addr: endpoint address for which endpoint object is required
107 *
108 */
109u8 cdns3_ep_addr_to_index(u8 ep_addr)
110{
111 return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
112}
113
114static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
115 struct cdns3_endpoint *priv_ep)
116{
117 int dma_index;
118
119 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
120
121 return dma_index / TRB_SIZE;
122}
123
124/**
125 * cdns3_next_request - returns next request from list
126 * @list: list containing requests
127 *
128 * Returns request or NULL if no requests in list
129 */
130struct usb_request *cdns3_next_request(struct list_head *list)
131{
132 return list_first_entry_or_null(list, struct usb_request, list);
133}
134
135/**
136 * cdns3_next_align_buf - returns next buffer from list
137 * @list: list containing buffers
138 *
139 * Returns buffer or NULL if no buffers in list
140 */
141static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
142{
143 return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
144}
145
146/**
147 * cdns3_next_priv_request - returns next request from list
148 * @list: list containing requests
149 *
150 * Returns request or NULL if no requests in list
151 */
152static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
153{
154 return list_first_entry_or_null(list, struct cdns3_request, list);
155}
156
157/**
158 * cdns3_select_ep - selects endpoint
159 * @priv_dev: extended gadget object
160 * @ep: endpoint address
161 */
162void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
163{
164 if (priv_dev->selected_ep == ep)
165 return;
166
167 priv_dev->selected_ep = ep;
168 writel(ep, &priv_dev->regs->ep_sel);
169}
170
171/**
172 * cdns3_get_tdl - gets current tdl for selected endpoint.
173 * @priv_dev: extended gadget object
174 *
175 * Before calling this function the appropriate endpoint must
176 * be selected by means of cdns3_select_ep function.
177 */
178static int cdns3_get_tdl(struct cdns3_device *priv_dev)
179{
180 if (priv_dev->dev_ver < DEV_VER_V3)
181 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
182 else
183 return readl(&priv_dev->regs->ep_tdl);
184}
185
186dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
187 struct cdns3_trb *trb)
188{
189 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
190
191 return priv_ep->trb_pool_dma + offset;
192}
193
194static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
195{
196 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
197
198 if (priv_ep->trb_pool) {
199 dma_pool_free(priv_dev->eps_dma_pool,
200 priv_ep->trb_pool, priv_ep->trb_pool_dma);
201 priv_ep->trb_pool = NULL;
202 }
203}
204
205/**
206 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
207 * @priv_ep: endpoint object
208 *
209 * Function will return 0 on success or -ENOMEM on allocation error
210 */
211int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
212{
213 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
214 int ring_size = TRB_RING_SIZE;
215 int num_trbs = ring_size / TRB_SIZE;
216 struct cdns3_trb *link_trb;
217
218 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
219 cdns3_free_trb_pool(priv_ep);
220
221 if (!priv_ep->trb_pool) {
222 priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,
223 GFP_ATOMIC,
224 &priv_ep->trb_pool_dma);
225
226 if (!priv_ep->trb_pool)
227 return -ENOMEM;
228
229 priv_ep->alloc_ring_size = ring_size;
230 }
231
232 memset(priv_ep->trb_pool, 0, ring_size);
233
234 priv_ep->num_trbs = num_trbs;
235
236 if (!priv_ep->num)
237 return 0;
238
239 /* Initialize the last TRB as Link TRB */
240 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
241
242 if (priv_ep->use_streams) {
243 /*
244 * For stream capable endpoints driver use single correct TRB.
245 * The last trb has zeroed cycle bit
246 */
247 link_trb->control = 0;
248 } else {
249 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
250 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
251 }
252 return 0;
253}
254
255/**
256 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
257 * @priv_ep: endpoint object
258 *
259 * Endpoint must be selected before call to this function
260 */
261static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
262{
263 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
264 int val;
265
266 trace_cdns3_halt(priv_ep, 1, 1);
267
268 writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
269 &priv_dev->regs->ep_cmd);
270
271 /* wait for DFLUSH cleared */
272 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
273 !(val & EP_CMD_DFLUSH), 1, 1000);
274 priv_ep->flags |= EP_STALLED;
275 priv_ep->flags &= ~EP_STALL_PENDING;
276}
277
278/**
279 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
280 * @priv_dev: extended gadget object
281 */
282void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
283{
284 int i;
285
286 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
287
288 cdns3_allow_enable_l1(priv_dev, 0);
289 priv_dev->hw_configured_flag = 0;
290 priv_dev->onchip_used_size = 0;
291 priv_dev->out_mem_is_allocated = 0;
292 priv_dev->wait_for_setup = 0;
293 priv_dev->using_streams = 0;
294
295 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
296 if (priv_dev->eps[i])
297 priv_dev->eps[i]->flags &= ~EP_CONFIGURED;
298}
299
300/**
301 * cdns3_ep_inc_trb - increment a trb index.
302 * @index: Pointer to the TRB index to increment.
303 * @cs: Cycle state
304 * @trb_in_seg: number of TRBs in segment
305 *
306 * The index should never point to the link TRB. After incrementing,
307 * if it is point to the link TRB, wrap around to the beginning and revert
308 * cycle state bit The
309 * link TRB is always at the last TRB entry.
310 */
311static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
312{
313 (*index)++;
314 if (*index == (trb_in_seg - 1)) {
315 *index = 0;
316 *cs ^= 1;
317 }
318}
319
320/**
321 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
322 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
323 */
324static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
325{
326 priv_ep->free_trbs--;
327 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
328}
329
330/**
331 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
332 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
333 */
334static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
335{
336 priv_ep->free_trbs++;
337 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
338}
339
340/**
341 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
342 * @priv_dev: Extended gadget object
343 * @enable: Enable/disable permit to transition to L1.
344 *
345 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
346 * then controller answer with ACK handshake.
347 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
348 * then controller answer with NYET handshake.
349 */
350void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
351{
352 if (enable)
353 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
354 else
355 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
356}
357
358enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
359{
360 u32 reg;
361
362 reg = readl(&priv_dev->regs->usb_sts);
363
364 if (DEV_SUPERSPEED(reg))
365 return USB_SPEED_SUPER;
366 else if (DEV_HIGHSPEED(reg))
367 return USB_SPEED_HIGH;
368 else if (DEV_FULLSPEED(reg))
369 return USB_SPEED_FULL;
370 else if (DEV_LOWSPEED(reg))
371 return USB_SPEED_LOW;
372 return USB_SPEED_UNKNOWN;
373}
374
375/**
376 * cdns3_start_all_request - add to ring all request not started
377 * @priv_dev: Extended gadget object
378 * @priv_ep: The endpoint for whom request will be started.
379 *
380 * Returns return ENOMEM if transfer ring i not enough TRBs to start
381 * all requests.
382 */
383static int cdns3_start_all_request(struct cdns3_device *priv_dev,
384 struct cdns3_endpoint *priv_ep)
385{
386 struct usb_request *request;
387 int ret = 0;
388 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
389
390 /*
391 * If the last pending transfer is INTERNAL
392 * OR streams are enabled for this endpoint
393 * do NOT start new transfer till the last one is pending
394 */
395 if (!pending_empty) {
396 struct cdns3_request *priv_req;
397
398 request = cdns3_next_request(&priv_ep->pending_req_list);
399 priv_req = to_cdns3_request(request);
400 if ((priv_req->flags & REQUEST_INTERNAL) ||
401 (priv_ep->flags & EP_TDLCHK_EN) ||
402 priv_ep->use_streams) {
403 dev_dbg(priv_dev->dev, "Blocking external request\n");
404 return ret;
405 }
406 }
407
408 while (!list_empty(&priv_ep->deferred_req_list)) {
409 request = cdns3_next_request(&priv_ep->deferred_req_list);
410
411 if (!priv_ep->use_streams) {
412 ret = cdns3_ep_run_transfer(priv_ep, request);
413 } else {
414 priv_ep->stream_sg_idx = 0;
415 ret = cdns3_ep_run_stream_transfer(priv_ep, request);
416 }
417 if (ret)
418 return ret;
419
420 list_move_tail(&request->list, &priv_ep->pending_req_list);
421 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
422 break;
423 }
424
425 priv_ep->flags &= ~EP_RING_FULL;
426 return ret;
427}
428
429/*
430 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
431 * driver try to detect whether endpoint need additional internal
432 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
433 * if before first DESCMISS interrupt the DMA will be armed.
434 */
435#define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
436 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
437 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
438 (reg) |= EP_STS_EN_DESCMISEN; \
439 } } while (0)
440
441static void __cdns3_descmiss_copy_data(struct usb_request *request,
442 struct usb_request *descmiss_req)
443{
444 int length = request->actual + descmiss_req->actual;
445 struct scatterlist *s = request->sg;
446
447 if (!s) {
448 if (length <= request->length) {
449 memcpy(&((u8 *)request->buf)[request->actual],
450 descmiss_req->buf,
451 descmiss_req->actual);
452 request->actual = length;
453 } else {
454 /* It should never occures */
455 request->status = -ENOMEM;
456 }
457 } else {
458 if (length <= sg_dma_len(s)) {
459 void *p = phys_to_virt(sg_dma_address(s));
460
461 memcpy(&((u8 *)p)[request->actual],
462 descmiss_req->buf,
463 descmiss_req->actual);
464 request->actual = length;
465 } else {
466 request->status = -ENOMEM;
467 }
468 }
469}
470
471/**
472 * cdns3_wa2_descmiss_copy_data - copy data from internal requests to
473 * request queued by class driver.
474 * @priv_ep: extended endpoint object
475 * @request: request object
476 */
477static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
478 struct usb_request *request)
479{
480 struct usb_request *descmiss_req;
481 struct cdns3_request *descmiss_priv_req;
482
483 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
484 int chunk_end;
485
486 descmiss_priv_req =
487 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
488 descmiss_req = &descmiss_priv_req->request;
489
490 /* driver can't touch pending request */
491 if (descmiss_priv_req->flags & REQUEST_PENDING)
492 break;
493
494 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
495 request->status = descmiss_req->status;
496 __cdns3_descmiss_copy_data(request, descmiss_req);
497 list_del_init(&descmiss_priv_req->list);
498 kfree(descmiss_req->buf);
499 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
500 --priv_ep->wa2_counter;
501
502 if (!chunk_end)
503 break;
504 }
505}
506
507static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
508 struct cdns3_endpoint *priv_ep,
509 struct cdns3_request *priv_req)
510{
511 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
512 priv_req->flags & REQUEST_INTERNAL) {
513 struct usb_request *req;
514
515 req = cdns3_next_request(&priv_ep->deferred_req_list);
516
517 priv_ep->descmis_req = NULL;
518
519 if (!req)
520 return NULL;
521
522 /* unmap the gadget request before copying data */
523 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
524 priv_ep->dir);
525
526 cdns3_wa2_descmiss_copy_data(priv_ep, req);
527 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
528 req->length != req->actual) {
529 /* wait for next part of transfer */
530 /* re-map the gadget request buffer*/
531 usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
532 usb_endpoint_dir_in(priv_ep->endpoint.desc));
533 return NULL;
534 }
535
536 if (req->status == -EINPROGRESS)
537 req->status = 0;
538
539 list_del_init(&req->list);
540 cdns3_start_all_request(priv_dev, priv_ep);
541 return req;
542 }
543
544 return &priv_req->request;
545}
546
547static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
548 struct cdns3_endpoint *priv_ep,
549 struct cdns3_request *priv_req)
550{
551 int deferred = 0;
552
553 /*
554 * If transfer was queued before DESCMISS appear than we
555 * can disable handling of DESCMISS interrupt. Driver assumes that it
556 * can disable special treatment for this endpoint.
557 */
558 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
559 u32 reg;
560
561 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
562 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
563 reg = readl(&priv_dev->regs->ep_sts_en);
564 reg &= ~EP_STS_EN_DESCMISEN;
565 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
566 writel(reg, &priv_dev->regs->ep_sts_en);
567 }
568
569 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
570 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
571 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
572
573 /*
574 * DESCMISS transfer has been finished, so data will be
575 * directly copied from internal allocated usb_request
576 * objects.
577 */
578 if (pending_empty && !descmiss_empty &&
579 !(priv_req->flags & REQUEST_INTERNAL)) {
580 cdns3_wa2_descmiss_copy_data(priv_ep,
581 &priv_req->request);
582
583 trace_cdns3_wa2(priv_ep, "get internal stored data");
584
585 list_add_tail(&priv_req->request.list,
586 &priv_ep->pending_req_list);
587 cdns3_gadget_giveback(priv_ep, priv_req,
588 priv_req->request.status);
589
590 /*
591 * Intentionally driver returns positive value as
592 * correct value. It informs that transfer has
593 * been finished.
594 */
595 return EINPROGRESS;
596 }
597
598 /*
599 * Driver will wait for completion DESCMISS transfer,
600 * before starts new, not DESCMISS transfer.
601 */
602 if (!pending_empty && !descmiss_empty) {
603 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
604 deferred = 1;
605 }
606
607 if (priv_req->flags & REQUEST_INTERNAL)
608 list_add_tail(&priv_req->list,
609 &priv_ep->wa2_descmiss_req_list);
610 }
611
612 return deferred;
613}
614
615static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
616{
617 struct cdns3_request *priv_req;
618
619 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
620 u8 chain;
621
622 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
623 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
624
625 trace_cdns3_wa2(priv_ep, "removes eldest request");
626
627 kfree(priv_req->request.buf);
628 list_del_init(&priv_req->list);
629 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
630 &priv_req->request);
631 --priv_ep->wa2_counter;
632
633 if (!chain)
634 break;
635 }
636}
637
638/**
639 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
640 * @priv_ep: extended gadget object
641 *
642 * This function is used only for WA2. For more information see Work around 2
643 * description.
644 */
645static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
646{
647 struct cdns3_request *priv_req;
648 struct usb_request *request;
649 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
650
651 /* check for pending transfer */
652 if (!pending_empty) {
653 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
654 return;
655 }
656
657 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
658 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
659 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
660 }
661
662 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
663
664 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
665 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
666 cdns3_wa2_remove_old_request(priv_ep);
667 }
668
669 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
670 GFP_ATOMIC);
671 if (!request)
672 goto err;
673
674 priv_req = to_cdns3_request(request);
675 priv_req->flags |= REQUEST_INTERNAL;
676
677 /* if this field is still assigned it indicate that transfer related
678 * with this request has not been finished yet. Driver in this
679 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
680 * flag to previous one. It will indicate that current request is
681 * part of the previous one.
682 */
683 if (priv_ep->descmis_req)
684 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
685
686 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
687 GFP_ATOMIC);
688 priv_ep->wa2_counter++;
689
690 if (!priv_req->request.buf) {
691 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
692 goto err;
693 }
694
695 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
696 priv_ep->descmis_req = priv_req;
697
698 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
699 &priv_ep->descmis_req->request,
700 GFP_ATOMIC);
701
702 return;
703
704err:
705 dev_err(priv_ep->cdns3_dev->dev,
706 "Failed: No sufficient memory for DESCMIS\n");
707}
708
709static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
710{
711 u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
712
713 if (tdl) {
714 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
715
716 writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
717 &priv_dev->regs->ep_cmd);
718 }
719}
720
721static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
722{
723 u32 ep_sts_reg;
724
725 /* select EP0-out */
726 cdns3_select_ep(priv_dev, 0);
727
728 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
729
730 if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
731 u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
732 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
733
734 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
735 outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
736 u8 pending_empty = list_empty(&outq_ep->pending_req_list);
737
738 if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
739 (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
740 !pending_empty) {
741 } else {
742 u32 ep_sts_en_reg;
743 u32 ep_cmd_reg;
744
745 cdns3_select_ep(priv_dev, outq_ep->num |
746 outq_ep->dir);
747 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
748 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
749
750 outq_ep->flags |= EP_TDLCHK_EN;
751 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
752 EP_CFG_TDL_CHK);
753
754 cdns3_wa2_enable_detection(priv_dev, outq_ep,
755 ep_sts_en_reg);
756 writel(ep_sts_en_reg,
757 &priv_dev->regs->ep_sts_en);
758 /* reset tdl value to zero */
759 cdns3_wa2_reset_tdl(priv_dev);
760 /*
761 * Memory barrier - Reset tdl before ringing the
762 * doorbell.
763 */
764 wmb();
765 if (EP_CMD_DRDY & ep_cmd_reg) {
766 trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
767
768 } else {
769 trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
770 /*
771 * ring doorbell to generate DESCMIS irq
772 */
773 writel(EP_CMD_DRDY,
774 &priv_dev->regs->ep_cmd);
775 }
776 }
777 }
778 }
779}
780
781/**
782 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
783 * @priv_ep: The endpoint to whom the request belongs to
784 * @priv_req: The request we're giving back
785 * @status: completion code for the request
786 *
787 * Must be called with controller's lock held and interrupts disabled. This
788 * function will unmap @req and call its ->complete() callback to notify upper
789 * layers that it has completed.
790 */
791void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
792 struct cdns3_request *priv_req,
793 int status)
794{
795 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
796 struct usb_request *request = &priv_req->request;
797
798 list_del_init(&request->list);
799
800 if (request->status == -EINPROGRESS)
801 request->status = status;
802
803 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
804 priv_ep->dir);
805
806 if ((priv_req->flags & REQUEST_UNALIGNED) &&
807 priv_ep->dir == USB_DIR_OUT && !request->status) {
808 /* Make DMA buffer CPU accessible */
809 dma_sync_single_for_cpu(priv_dev->sysdev,
810 priv_req->aligned_buf->dma,
811 priv_req->aligned_buf->size,
812 priv_req->aligned_buf->dir);
813 memcpy(request->buf, priv_req->aligned_buf->buf,
814 request->length);
815 }
816
817 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
818 /* All TRBs have finished, clear the counter */
819 priv_req->finished_trb = 0;
820 trace_cdns3_gadget_giveback(priv_req);
821
822 if (priv_dev->dev_ver < DEV_VER_V2) {
823 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
824 priv_req);
825 if (!request)
826 return;
827 }
828
829 if (request->complete) {
830 spin_unlock(&priv_dev->lock);
831 usb_gadget_giveback_request(&priv_ep->endpoint,
832 request);
833 spin_lock(&priv_dev->lock);
834 }
835
836 if (request->buf == priv_dev->zlp_buf)
837 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
838}
839
840static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
841{
842 /* Work around for stale data address in TRB*/
843 if (priv_ep->wa1_set) {
844 trace_cdns3_wa1(priv_ep, "restore cycle bit");
845
846 priv_ep->wa1_set = 0;
847 priv_ep->wa1_trb_index = 0xFFFF;
848 if (priv_ep->wa1_cycle_bit) {
849 priv_ep->wa1_trb->control =
850 priv_ep->wa1_trb->control | cpu_to_le32(0x1);
851 } else {
852 priv_ep->wa1_trb->control =
853 priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
854 }
855 }
856}
857
858static void cdns3_free_aligned_request_buf(struct work_struct *work)
859{
860 struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
861 aligned_buf_wq);
862 struct cdns3_aligned_buf *buf, *tmp;
863 unsigned long flags;
864
865 spin_lock_irqsave(&priv_dev->lock, flags);
866
867 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
868 if (!buf->in_use) {
869 list_del(&buf->list);
870
871 /*
872 * Re-enable interrupts to free DMA capable memory.
873 * Driver can't free this memory with disabled
874 * interrupts.
875 */
876 spin_unlock_irqrestore(&priv_dev->lock, flags);
877 dma_free_noncoherent(priv_dev->sysdev, buf->size,
878 buf->buf, buf->dma, buf->dir);
879 kfree(buf);
880 spin_lock_irqsave(&priv_dev->lock, flags);
881 }
882 }
883
884 spin_unlock_irqrestore(&priv_dev->lock, flags);
885}
886
887static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
888{
889 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
890 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
891 struct cdns3_aligned_buf *buf;
892
893 /* check if buffer is aligned to 8. */
894 if (!((uintptr_t)priv_req->request.buf & 0x7))
895 return 0;
896
897 buf = priv_req->aligned_buf;
898
899 if (!buf || priv_req->request.length > buf->size) {
900 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
901 if (!buf)
902 return -ENOMEM;
903
904 buf->size = priv_req->request.length;
905 buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ?
906 DMA_TO_DEVICE : DMA_FROM_DEVICE;
907
908 buf->buf = dma_alloc_noncoherent(priv_dev->sysdev,
909 buf->size,
910 &buf->dma,
911 buf->dir,
912 GFP_ATOMIC);
913 if (!buf->buf) {
914 kfree(buf);
915 return -ENOMEM;
916 }
917
918 if (priv_req->aligned_buf) {
919 trace_cdns3_free_aligned_request(priv_req);
920 priv_req->aligned_buf->in_use = 0;
921 queue_work(system_freezable_wq,
922 &priv_dev->aligned_buf_wq);
923 }
924
925 buf->in_use = 1;
926 priv_req->aligned_buf = buf;
927
928 list_add_tail(&buf->list,
929 &priv_dev->aligned_buf_list);
930 }
931
932 if (priv_ep->dir == USB_DIR_IN) {
933 /* Make DMA buffer CPU accessible */
934 dma_sync_single_for_cpu(priv_dev->sysdev,
935 buf->dma, buf->size, buf->dir);
936 memcpy(buf->buf, priv_req->request.buf,
937 priv_req->request.length);
938 }
939
940 /* Transfer DMA buffer ownership back to device */
941 dma_sync_single_for_device(priv_dev->sysdev,
942 buf->dma, buf->size, buf->dir);
943
944 priv_req->flags |= REQUEST_UNALIGNED;
945 trace_cdns3_prepare_aligned_request(priv_req);
946
947 return 0;
948}
949
950static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
951 struct cdns3_trb *trb)
952{
953 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
954
955 if (!priv_ep->wa1_set) {
956 u32 doorbell;
957
958 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
959
960 if (doorbell) {
961 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
962 priv_ep->wa1_set = 1;
963 priv_ep->wa1_trb = trb;
964 priv_ep->wa1_trb_index = priv_ep->enqueue;
965 trace_cdns3_wa1(priv_ep, "set guard");
966 return 0;
967 }
968 }
969 return 1;
970}
971
972static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
973 struct cdns3_endpoint *priv_ep)
974{
975 int dma_index;
976 u32 doorbell;
977
978 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
979 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
980
981 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
982 cdns3_wa1_restore_cycle_bit(priv_ep);
983}
984
985static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
986 struct usb_request *request)
987{
988 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
989 struct cdns3_request *priv_req;
990 struct cdns3_trb *trb;
991 dma_addr_t trb_dma;
992 int address;
993 u32 control;
994 u32 length;
995 u32 tdl;
996 unsigned int sg_idx = priv_ep->stream_sg_idx;
997
998 priv_req = to_cdns3_request(request);
999 address = priv_ep->endpoint.desc->bEndpointAddress;
1000
1001 priv_ep->flags |= EP_PENDING_REQUEST;
1002
1003 /* must allocate buffer aligned to 8 */
1004 if (priv_req->flags & REQUEST_UNALIGNED)
1005 trb_dma = priv_req->aligned_buf->dma;
1006 else
1007 trb_dma = request->dma;
1008
1009 /* For stream capable endpoints driver use only single TD. */
1010 trb = priv_ep->trb_pool + priv_ep->enqueue;
1011 priv_req->start_trb = priv_ep->enqueue;
1012 priv_req->end_trb = priv_req->start_trb;
1013 priv_req->trb = trb;
1014
1015 cdns3_select_ep(priv_ep->cdns3_dev, address);
1016
1017 control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1018 TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1019
1020 if (!request->num_sgs) {
1021 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1022 length = request->length;
1023 } else {
1024 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
1025 length = request->sg[sg_idx].length;
1026 }
1027
1028 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1029
1030 trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
1031
1032 /*
1033 * For DEV_VER_V2 controller version we have enabled
1034 * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1035 * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1036 */
1037 if (priv_dev->dev_ver >= DEV_VER_V2) {
1038 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1039 trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl));
1040 }
1041 priv_req->flags |= REQUEST_PENDING;
1042
1043 trb->control = cpu_to_le32(control);
1044
1045 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1046
1047 /*
1048 * Memory barrier - Cycle Bit must be set before trb->length and
1049 * trb->buffer fields.
1050 */
1051 wmb();
1052
1053 /* always first element */
1054 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1055 &priv_dev->regs->ep_traddr);
1056
1057 if (!(priv_ep->flags & EP_STALLED)) {
1058 trace_cdns3_ring(priv_ep);
1059 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1060 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1061
1062 priv_ep->prime_flag = false;
1063
1064 /*
1065 * Controller version DEV_VER_V2 tdl calculation
1066 * is based on TRB
1067 */
1068
1069 if (priv_dev->dev_ver < DEV_VER_V2)
1070 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1071 &priv_dev->regs->ep_cmd);
1072 else if (priv_dev->dev_ver > DEV_VER_V2)
1073 writel(tdl, &priv_dev->regs->ep_tdl);
1074
1075 priv_ep->last_stream_id = priv_req->request.stream_id;
1076 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1077 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1078 EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1079
1080 trace_cdns3_doorbell_epx(priv_ep->name,
1081 readl(&priv_dev->regs->ep_traddr));
1082 }
1083
1084 /* WORKAROUND for transition to L0 */
1085 __cdns3_gadget_wakeup(priv_dev);
1086
1087 return 0;
1088}
1089
1090static void cdns3_rearm_drdy_if_needed(struct cdns3_endpoint *priv_ep)
1091{
1092 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1093
1094 if (priv_dev->dev_ver < DEV_VER_V3)
1095 return;
1096
1097 if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) {
1098 writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts);
1099 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1100 }
1101}
1102
1103/**
1104 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1105 * @priv_ep: endpoint object
1106 * @request: request object
1107 *
1108 * Returns zero on success or negative value on failure
1109 */
1110static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1111 struct usb_request *request)
1112{
1113 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1114 struct cdns3_request *priv_req;
1115 struct cdns3_trb *trb;
1116 struct cdns3_trb *link_trb = NULL;
1117 dma_addr_t trb_dma;
1118 u32 togle_pcs = 1;
1119 int sg_iter = 0;
1120 int num_trb;
1121 int address;
1122 u32 control;
1123 int pcs;
1124 u16 total_tdl = 0;
1125 struct scatterlist *s = NULL;
1126 bool sg_supported = !!(request->num_mapped_sgs);
1127
1128 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1129 num_trb = priv_ep->interval;
1130 else
1131 num_trb = sg_supported ? request->num_mapped_sgs : 1;
1132
1133 if (num_trb > priv_ep->free_trbs) {
1134 priv_ep->flags |= EP_RING_FULL;
1135 return -ENOBUFS;
1136 }
1137
1138 priv_req = to_cdns3_request(request);
1139 address = priv_ep->endpoint.desc->bEndpointAddress;
1140
1141 priv_ep->flags |= EP_PENDING_REQUEST;
1142
1143 /* must allocate buffer aligned to 8 */
1144 if (priv_req->flags & REQUEST_UNALIGNED)
1145 trb_dma = priv_req->aligned_buf->dma;
1146 else
1147 trb_dma = request->dma;
1148
1149 trb = priv_ep->trb_pool + priv_ep->enqueue;
1150 priv_req->start_trb = priv_ep->enqueue;
1151 priv_req->trb = trb;
1152
1153 cdns3_select_ep(priv_ep->cdns3_dev, address);
1154
1155 /* prepare ring */
1156 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
1157 int doorbell, dma_index;
1158 u32 ch_bit = 0;
1159
1160 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1161 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1162
1163 /* Driver can't update LINK TRB if it is current processed. */
1164 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1165 priv_ep->flags |= EP_DEFERRED_DRDY;
1166 return -ENOBUFS;
1167 }
1168
1169 /*updating C bt in Link TRB before starting DMA*/
1170 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1171 /*
1172 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1173 * that DMA stuck at the LINK TRB.
1174 * On the other hand, removing TRB_CHAIN for longer TRs for
1175 * epXout cause that DMA stuck after handling LINK TRB.
1176 * To eliminate this strange behavioral driver set TRB_CHAIN
1177 * bit only for TR size > 2.
1178 */
1179 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1180 TRBS_PER_SEGMENT > 2)
1181 ch_bit = TRB_CHAIN;
1182
1183 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1184 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
1185 }
1186
1187 if (priv_dev->dev_ver <= DEV_VER_V2)
1188 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1189
1190 if (sg_supported)
1191 s = request->sg;
1192
1193 /* set incorrect Cycle Bit for first trb*/
1194 control = priv_ep->pcs ? 0 : TRB_CYCLE;
1195 trb->length = 0;
1196 if (priv_dev->dev_ver >= DEV_VER_V2) {
1197 u16 td_size;
1198
1199 td_size = DIV_ROUND_UP(request->length,
1200 priv_ep->endpoint.maxpacket);
1201 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1202 trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
1203 else
1204 control |= TRB_TDL_HS_SIZE(td_size);
1205 }
1206
1207 do {
1208 u32 length;
1209
1210 /* fill TRB */
1211 control |= TRB_TYPE(TRB_NORMAL);
1212 if (sg_supported) {
1213 trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
1214 length = sg_dma_len(s);
1215 } else {
1216 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1217 length = request->length;
1218 }
1219
1220 if (priv_ep->flags & EP_TDLCHK_EN)
1221 total_tdl += DIV_ROUND_UP(length,
1222 priv_ep->endpoint.maxpacket);
1223
1224 trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) |
1225 TRB_LEN(length));
1226 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1227
1228 /*
1229 * first trb should be prepared as last to avoid processing
1230 * transfer to early
1231 */
1232 if (sg_iter != 0)
1233 control |= pcs;
1234
1235 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
1236 control |= TRB_IOC | TRB_ISP;
1237 } else {
1238 /* for last element in TD or in SG list */
1239 if (sg_iter == (num_trb - 1) && sg_iter != 0)
1240 control |= pcs | TRB_IOC | TRB_ISP;
1241 }
1242
1243 if (sg_iter)
1244 trb->control = cpu_to_le32(control);
1245 else
1246 priv_req->trb->control = cpu_to_le32(control);
1247
1248 if (sg_supported) {
1249 trb->control |= cpu_to_le32(TRB_ISP);
1250 /* Don't set chain bit for last TRB */
1251 if (sg_iter < num_trb - 1)
1252 trb->control |= cpu_to_le32(TRB_CHAIN);
1253
1254 s = sg_next(s);
1255 }
1256
1257 control = 0;
1258 ++sg_iter;
1259 priv_req->end_trb = priv_ep->enqueue;
1260 cdns3_ep_inc_enq(priv_ep);
1261 trb = priv_ep->trb_pool + priv_ep->enqueue;
1262 trb->length = 0;
1263 } while (sg_iter < num_trb);
1264
1265 trb = priv_req->trb;
1266
1267 priv_req->flags |= REQUEST_PENDING;
1268 priv_req->num_of_trb = num_trb;
1269
1270 if (sg_iter == 1)
1271 trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP);
1272
1273 if (priv_dev->dev_ver < DEV_VER_V2 &&
1274 (priv_ep->flags & EP_TDLCHK_EN)) {
1275 u16 tdl = total_tdl;
1276 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1277
1278 if (tdl > EP_CMD_TDL_MAX) {
1279 tdl = EP_CMD_TDL_MAX;
1280 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1281 }
1282
1283 if (old_tdl < tdl) {
1284 tdl -= old_tdl;
1285 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1286 &priv_dev->regs->ep_cmd);
1287 }
1288 }
1289
1290 /*
1291 * Memory barrier - cycle bit must be set before other filds in trb.
1292 */
1293 wmb();
1294
1295 /* give the TD to the consumer*/
1296 if (togle_pcs)
1297 trb->control = trb->control ^ cpu_to_le32(1);
1298
1299 if (priv_dev->dev_ver <= DEV_VER_V2)
1300 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1301
1302 if (num_trb > 1) {
1303 int i = 0;
1304
1305 while (i < num_trb) {
1306 trace_cdns3_prepare_trb(priv_ep, trb + i);
1307 if (trb + i == link_trb) {
1308 trb = priv_ep->trb_pool;
1309 num_trb = num_trb - i;
1310 i = 0;
1311 } else {
1312 i++;
1313 }
1314 }
1315 } else {
1316 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1317 }
1318
1319 /*
1320 * Memory barrier - Cycle Bit must be set before trb->length and
1321 * trb->buffer fields.
1322 */
1323 wmb();
1324
1325 /*
1326 * For DMULT mode we can set address to transfer ring only once after
1327 * enabling endpoint.
1328 */
1329 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1330 /*
1331 * Until SW is not ready to handle the OUT transfer the ISO OUT
1332 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1333 * EP_CFG_ENABLE must be set before updating ep_traddr.
1334 */
1335 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
1336 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1337 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1338 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1339 EP_CFG_ENABLE);
1340 }
1341
1342 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1343 priv_req->start_trb * TRB_SIZE),
1344 &priv_dev->regs->ep_traddr);
1345
1346 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1347 }
1348
1349 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1350 trace_cdns3_ring(priv_ep);
1351 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1352 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1353 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1354 cdns3_rearm_drdy_if_needed(priv_ep);
1355 trace_cdns3_doorbell_epx(priv_ep->name,
1356 readl(&priv_dev->regs->ep_traddr));
1357 }
1358
1359 /* WORKAROUND for transition to L0 */
1360 __cdns3_gadget_wakeup(priv_dev);
1361
1362 return 0;
1363}
1364
1365void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1366{
1367 struct cdns3_endpoint *priv_ep;
1368 struct usb_ep *ep;
1369
1370 if (priv_dev->hw_configured_flag)
1371 return;
1372
1373 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1374
1375 cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1376 USB_CONF_U1EN | USB_CONF_U2EN);
1377
1378 priv_dev->hw_configured_flag = 1;
1379
1380 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1381 if (ep->enabled) {
1382 priv_ep = ep_to_cdns3_ep(ep);
1383 cdns3_start_all_request(priv_dev, priv_ep);
1384 }
1385 }
1386
1387 cdns3_allow_enable_l1(priv_dev, 1);
1388}
1389
1390/**
1391 * cdns3_trb_handled - check whether trb has been handled by DMA
1392 *
1393 * @priv_ep: extended endpoint object.
1394 * @priv_req: request object for checking
1395 *
1396 * Endpoint must be selected before invoking this function.
1397 *
1398 * Returns false if request has not been handled by DMA, else returns true.
1399 *
1400 * SR - start ring
1401 * ER - end ring
1402 * DQ = priv_ep->dequeue - dequeue position
1403 * EQ = priv_ep->enqueue - enqueue position
1404 * ST = priv_req->start_trb - index of first TRB in transfer ring
1405 * ET = priv_req->end_trb - index of last TRB in transfer ring
1406 * CI = current_index - index of processed TRB by DMA.
1407 *
1408 * As first step, we check if the TRB between the ST and ET.
1409 * Then, we check if cycle bit for index priv_ep->dequeue
1410 * is correct.
1411 *
1412 * some rules:
1413 * 1. priv_ep->dequeue never equals to current_index.
1414 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1415 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1416 * and priv_ep->free_trbs is zero.
1417 * This case indicate that TR is full.
1418 *
1419 * At below two cases, the request have been handled.
1420 * Case 1 - priv_ep->dequeue < current_index
1421 * SR ... EQ ... DQ ... CI ... ER
1422 * SR ... DQ ... CI ... EQ ... ER
1423 *
1424 * Case 2 - priv_ep->dequeue > current_index
1425 * This situation takes place when CI go through the LINK TRB at the end of
1426 * transfer ring.
1427 * SR ... CI ... EQ ... DQ ... ER
1428 */
1429static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1430 struct cdns3_request *priv_req)
1431{
1432 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1433 struct cdns3_trb *trb;
1434 int current_index = 0;
1435 int handled = 0;
1436 int doorbell;
1437
1438 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1439 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1440
1441 /* current trb doesn't belong to this request */
1442 if (priv_req->start_trb < priv_req->end_trb) {
1443 if (priv_ep->dequeue > priv_req->end_trb)
1444 goto finish;
1445
1446 if (priv_ep->dequeue < priv_req->start_trb)
1447 goto finish;
1448 }
1449
1450 if ((priv_req->start_trb > priv_req->end_trb) &&
1451 (priv_ep->dequeue > priv_req->end_trb) &&
1452 (priv_ep->dequeue < priv_req->start_trb))
1453 goto finish;
1454
1455 if ((priv_req->start_trb == priv_req->end_trb) &&
1456 (priv_ep->dequeue != priv_req->end_trb))
1457 goto finish;
1458
1459 trb = &priv_ep->trb_pool[priv_ep->dequeue];
1460
1461 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1462 goto finish;
1463
1464 if (doorbell == 1 && current_index == priv_ep->dequeue)
1465 goto finish;
1466
1467 /* The corner case for TRBS_PER_SEGMENT equal 2). */
1468 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1469 handled = 1;
1470 goto finish;
1471 }
1472
1473 if (priv_ep->enqueue == priv_ep->dequeue &&
1474 priv_ep->free_trbs == 0) {
1475 handled = 1;
1476 } else if (priv_ep->dequeue < current_index) {
1477 if ((current_index == (priv_ep->num_trbs - 1)) &&
1478 !priv_ep->dequeue)
1479 goto finish;
1480
1481 handled = 1;
1482 } else if (priv_ep->dequeue > current_index) {
1483 handled = 1;
1484 }
1485
1486finish:
1487 trace_cdns3_request_handled(priv_req, current_index, handled);
1488
1489 return handled;
1490}
1491
1492static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1493 struct cdns3_endpoint *priv_ep)
1494{
1495 struct cdns3_request *priv_req;
1496 struct usb_request *request;
1497 struct cdns3_trb *trb;
1498 bool request_handled = false;
1499 bool transfer_end = false;
1500
1501 while (!list_empty(&priv_ep->pending_req_list)) {
1502 request = cdns3_next_request(&priv_ep->pending_req_list);
1503 priv_req = to_cdns3_request(request);
1504
1505 trb = priv_ep->trb_pool + priv_ep->dequeue;
1506
1507 /* The TRB was changed as link TRB, and the request was handled at ep_dequeue */
1508 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
1509 trace_cdns3_complete_trb(priv_ep, trb);
1510 cdns3_ep_inc_deq(priv_ep);
1511 trb = priv_ep->trb_pool + priv_ep->dequeue;
1512 }
1513
1514 if (!request->stream_id) {
1515 /* Re-select endpoint. It could be changed by other CPU
1516 * during handling usb_gadget_giveback_request.
1517 */
1518 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1519
1520 while (cdns3_trb_handled(priv_ep, priv_req)) {
1521 priv_req->finished_trb++;
1522 if (priv_req->finished_trb >= priv_req->num_of_trb)
1523 request_handled = true;
1524
1525 trb = priv_ep->trb_pool + priv_ep->dequeue;
1526 trace_cdns3_complete_trb(priv_ep, trb);
1527
1528 if (!transfer_end)
1529 request->actual +=
1530 TRB_LEN(le32_to_cpu(trb->length));
1531
1532 if (priv_req->num_of_trb > 1 &&
1533 le32_to_cpu(trb->control) & TRB_SMM &&
1534 le32_to_cpu(trb->control) & TRB_CHAIN)
1535 transfer_end = true;
1536
1537 cdns3_ep_inc_deq(priv_ep);
1538 }
1539
1540 if (request_handled) {
1541 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1542 request_handled = false;
1543 transfer_end = false;
1544 } else {
1545 goto prepare_next_td;
1546 }
1547
1548 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1549 TRBS_PER_SEGMENT == 2)
1550 break;
1551 } else {
1552 /* Re-select endpoint. It could be changed by other CPU
1553 * during handling usb_gadget_giveback_request.
1554 */
1555 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1556
1557 trb = priv_ep->trb_pool;
1558 trace_cdns3_complete_trb(priv_ep, trb);
1559
1560 if (trb != priv_req->trb)
1561 dev_warn(priv_dev->dev,
1562 "request_trb=0x%p, queue_trb=0x%p\n",
1563 priv_req->trb, trb);
1564
1565 request->actual += TRB_LEN(le32_to_cpu(trb->length));
1566
1567 if (!request->num_sgs ||
1568 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1569 priv_ep->stream_sg_idx = 0;
1570 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1571 } else {
1572 priv_ep->stream_sg_idx++;
1573 cdns3_ep_run_stream_transfer(priv_ep, request);
1574 }
1575 break;
1576 }
1577 }
1578 priv_ep->flags &= ~EP_PENDING_REQUEST;
1579
1580prepare_next_td:
1581 if (!(priv_ep->flags & EP_STALLED) &&
1582 !(priv_ep->flags & EP_STALL_PENDING))
1583 cdns3_start_all_request(priv_dev, priv_ep);
1584}
1585
1586void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1587{
1588 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1589
1590 cdns3_wa1_restore_cycle_bit(priv_ep);
1591
1592 if (rearm) {
1593 trace_cdns3_ring(priv_ep);
1594
1595 /* Cycle Bit must be updated before arming DMA. */
1596 wmb();
1597 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1598
1599 __cdns3_gadget_wakeup(priv_dev);
1600
1601 trace_cdns3_doorbell_epx(priv_ep->name,
1602 readl(&priv_dev->regs->ep_traddr));
1603 }
1604}
1605
1606static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1607{
1608 u16 tdl = priv_ep->pending_tdl;
1609 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1610
1611 if (tdl > EP_CMD_TDL_MAX) {
1612 tdl = EP_CMD_TDL_MAX;
1613 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1614 } else {
1615 priv_ep->pending_tdl = 0;
1616 }
1617
1618 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1619}
1620
1621/**
1622 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1623 * @priv_ep: endpoint object
1624 *
1625 * Returns 0
1626 */
1627static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1628{
1629 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1630 u32 ep_sts_reg;
1631 struct usb_request *deferred_request;
1632 struct usb_request *pending_request;
1633 u32 tdl = 0;
1634
1635 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1636
1637 trace_cdns3_epx_irq(priv_dev, priv_ep);
1638
1639 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1640 writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1641
1642 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1643 bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1644
1645 tdl = cdns3_get_tdl(priv_dev);
1646
1647 /*
1648 * Continue the previous transfer:
1649 * There is some racing between ERDY and PRIME. The device send
1650 * ERDY and almost in the same time Host send PRIME. It cause
1651 * that host ignore the ERDY packet and driver has to send it
1652 * again.
1653 */
1654 if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) ||
1655 EP_STS_HOSTPP(ep_sts_reg))) {
1656 writel(EP_CMD_ERDY |
1657 EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1658 &priv_dev->regs->ep_cmd);
1659 ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1660 } else {
1661 priv_ep->prime_flag = true;
1662
1663 pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1664 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1665
1666 if (deferred_request && !pending_request) {
1667 cdns3_start_all_request(priv_dev, priv_ep);
1668 }
1669 }
1670 }
1671
1672 if (ep_sts_reg & EP_STS_TRBERR) {
1673 if (priv_ep->flags & EP_STALL_PENDING &&
1674 !(ep_sts_reg & EP_STS_DESCMIS &&
1675 priv_dev->dev_ver < DEV_VER_V2)) {
1676 cdns3_ep_stall_flush(priv_ep);
1677 }
1678
1679 /*
1680 * For isochronous transfer driver completes request on
1681 * IOC or on TRBERR. IOC appears only when device receive
1682 * OUT data packet. If host disable stream or lost some packet
1683 * then the only way to finish all queued transfer is to do it
1684 * on TRBERR event.
1685 */
1686 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1687 !priv_ep->wa1_set) {
1688 if (!priv_ep->dir) {
1689 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1690
1691 ep_cfg &= ~EP_CFG_ENABLE;
1692 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1693 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1694 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
1695 }
1696 cdns3_transfer_completed(priv_dev, priv_ep);
1697 } else if (!(priv_ep->flags & EP_STALLED) &&
1698 !(priv_ep->flags & EP_STALL_PENDING)) {
1699 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1700 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1701 cdns3_start_all_request(priv_dev, priv_ep);
1702 } else {
1703 cdns3_rearm_transfer(priv_ep,
1704 priv_ep->wa1_set);
1705 }
1706 }
1707 }
1708
1709 if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1710 (ep_sts_reg & EP_STS_IOT)) {
1711 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1712 if (ep_sts_reg & EP_STS_ISP)
1713 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1714 else
1715 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1716 }
1717
1718 if (!priv_ep->use_streams) {
1719 if ((ep_sts_reg & EP_STS_IOC) ||
1720 (ep_sts_reg & EP_STS_ISP)) {
1721 cdns3_transfer_completed(priv_dev, priv_ep);
1722 } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1723 priv_ep->pending_tdl) {
1724 /* handle IOT with pending tdl */
1725 cdns3_reprogram_tdl(priv_ep);
1726 }
1727 } else if (priv_ep->dir == USB_DIR_OUT) {
1728 priv_ep->ep_sts_pending |= ep_sts_reg;
1729 } else if (ep_sts_reg & EP_STS_IOT) {
1730 cdns3_transfer_completed(priv_dev, priv_ep);
1731 }
1732 }
1733
1734 /*
1735 * MD_EXIT interrupt sets when stream capable endpoint exits
1736 * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1737 */
1738 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1739 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1740 priv_ep->ep_sts_pending = 0;
1741 cdns3_transfer_completed(priv_dev, priv_ep);
1742 }
1743
1744 /*
1745 * WA2: this condition should only be meet when
1746 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1747 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1748 * In other cases this interrupt will be disabled.
1749 */
1750 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1751 !(priv_ep->flags & EP_STALLED))
1752 cdns3_wa2_descmissing_packet(priv_ep);
1753
1754 return 0;
1755}
1756
1757static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1758{
1759 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)
1760 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1761}
1762
1763/**
1764 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1765 * @priv_dev: extended gadget object
1766 * @usb_ists: bitmap representation of device's reported interrupts
1767 * (usb_ists register value)
1768 */
1769static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1770 u32 usb_ists)
1771__must_hold(&priv_dev->lock)
1772{
1773 int speed = 0;
1774
1775 trace_cdns3_usb_irq(priv_dev, usb_ists);
1776 if (usb_ists & USB_ISTS_L1ENTI) {
1777 /*
1778 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1779 * from L1. To fix it, if any DMA transfer is pending driver
1780 * must starts driving resume signal immediately.
1781 */
1782 if (readl(&priv_dev->regs->drbl))
1783 __cdns3_gadget_wakeup(priv_dev);
1784 }
1785
1786 /* Connection detected */
1787 if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1788 speed = cdns3_get_speed(priv_dev);
1789 priv_dev->gadget.speed = speed;
1790 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1791 cdns3_ep0_config(priv_dev);
1792 }
1793
1794 /* Disconnection detected */
1795 if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1796 spin_unlock(&priv_dev->lock);
1797 cdns3_disconnect_gadget(priv_dev);
1798 spin_lock(&priv_dev->lock);
1799 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1800 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1801 cdns3_hw_reset_eps_config(priv_dev);
1802 }
1803
1804 if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1805 if (priv_dev->gadget_driver &&
1806 priv_dev->gadget_driver->suspend) {
1807 spin_unlock(&priv_dev->lock);
1808 priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1809 spin_lock(&priv_dev->lock);
1810 }
1811 }
1812
1813 if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1814 if (priv_dev->gadget_driver &&
1815 priv_dev->gadget_driver->resume) {
1816 spin_unlock(&priv_dev->lock);
1817 priv_dev->gadget_driver->resume(&priv_dev->gadget);
1818 spin_lock(&priv_dev->lock);
1819 }
1820 }
1821
1822 /* reset*/
1823 if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1824 if (priv_dev->gadget_driver) {
1825 spin_unlock(&priv_dev->lock);
1826 usb_gadget_udc_reset(&priv_dev->gadget,
1827 priv_dev->gadget_driver);
1828 spin_lock(&priv_dev->lock);
1829
1830 /*read again to check the actual speed*/
1831 speed = cdns3_get_speed(priv_dev);
1832 priv_dev->gadget.speed = speed;
1833 cdns3_hw_reset_eps_config(priv_dev);
1834 cdns3_ep0_config(priv_dev);
1835 }
1836 }
1837}
1838
1839/**
1840 * cdns3_device_irq_handler - interrupt handler for device part of controller
1841 *
1842 * @irq: irq number for cdns3 core device
1843 * @data: structure of cdns3
1844 *
1845 * Returns IRQ_HANDLED or IRQ_NONE
1846 */
1847static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1848{
1849 struct cdns3_device *priv_dev = data;
1850 struct cdns *cdns = dev_get_drvdata(priv_dev->dev);
1851 irqreturn_t ret = IRQ_NONE;
1852 u32 reg;
1853
1854 if (cdns->in_lpm)
1855 return ret;
1856
1857 /* check USB device interrupt */
1858 reg = readl(&priv_dev->regs->usb_ists);
1859 if (reg) {
1860 /* After masking interrupts the new interrupts won't be
1861 * reported in usb_ists/ep_ists. In order to not lose some
1862 * of them driver disables only detected interrupts.
1863 * They will be enabled ASAP after clearing source of
1864 * interrupt. This an unusual behavior only applies to
1865 * usb_ists register.
1866 */
1867 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1868 /* mask deferred interrupt. */
1869 writel(reg, &priv_dev->regs->usb_ien);
1870 ret = IRQ_WAKE_THREAD;
1871 }
1872
1873 /* check endpoint interrupt */
1874 reg = readl(&priv_dev->regs->ep_ists);
1875 if (reg) {
1876 writel(0, &priv_dev->regs->ep_ien);
1877 ret = IRQ_WAKE_THREAD;
1878 }
1879
1880 return ret;
1881}
1882
1883/**
1884 * cdns3_device_thread_irq_handler - interrupt handler for device part
1885 * of controller
1886 *
1887 * @irq: irq number for cdns3 core device
1888 * @data: structure of cdns3
1889 *
1890 * Returns IRQ_HANDLED or IRQ_NONE
1891 */
1892static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1893{
1894 struct cdns3_device *priv_dev = data;
1895 irqreturn_t ret = IRQ_NONE;
1896 unsigned long flags;
1897 unsigned int bit;
1898 unsigned long reg;
1899
1900 spin_lock_irqsave(&priv_dev->lock, flags);
1901
1902 reg = readl(&priv_dev->regs->usb_ists);
1903 if (reg) {
1904 writel(reg, &priv_dev->regs->usb_ists);
1905 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1906 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1907 ret = IRQ_HANDLED;
1908 }
1909
1910 reg = readl(&priv_dev->regs->ep_ists);
1911
1912 /* handle default endpoint OUT */
1913 if (reg & EP_ISTS_EP_OUT0) {
1914 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1915 ret = IRQ_HANDLED;
1916 }
1917
1918 /* handle default endpoint IN */
1919 if (reg & EP_ISTS_EP_IN0) {
1920 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1921 ret = IRQ_HANDLED;
1922 }
1923
1924 /* check if interrupt from non default endpoint, if no exit */
1925 reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1926 if (!reg)
1927 goto irqend;
1928
1929 for_each_set_bit(bit, ®,
1930 sizeof(u32) * BITS_PER_BYTE) {
1931 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1932 ret = IRQ_HANDLED;
1933 }
1934
1935 if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1936 cdns3_wa2_check_outq_status(priv_dev);
1937
1938irqend:
1939 writel(~0, &priv_dev->regs->ep_ien);
1940 spin_unlock_irqrestore(&priv_dev->lock, flags);
1941
1942 return ret;
1943}
1944
1945/**
1946 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1947 *
1948 * The real reservation will occur during write to EP_CFG register,
1949 * this function is used to check if the 'size' reservation is allowed.
1950 *
1951 * @priv_dev: extended gadget object
1952 * @size: the size (KB) for EP would like to allocate
1953 * @is_in: endpoint direction
1954 *
1955 * Return 0 if the required size can met or negative value on failure
1956 */
1957static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1958 int size, int is_in)
1959{
1960 int remained;
1961
1962 /* 2KB are reserved for EP0*/
1963 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1964
1965 if (is_in) {
1966 if (remained < size)
1967 return -EPERM;
1968
1969 priv_dev->onchip_used_size += size;
1970 } else {
1971 int required;
1972
1973 /**
1974 * ALL OUT EPs are shared the same chunk onchip memory, so
1975 * driver checks if it already has assigned enough buffers
1976 */
1977 if (priv_dev->out_mem_is_allocated >= size)
1978 return 0;
1979
1980 required = size - priv_dev->out_mem_is_allocated;
1981
1982 if (required > remained)
1983 return -EPERM;
1984
1985 priv_dev->out_mem_is_allocated += required;
1986 priv_dev->onchip_used_size += required;
1987 }
1988
1989 return 0;
1990}
1991
1992static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1993 struct cdns3_endpoint *priv_ep)
1994{
1995 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1996
1997 /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1998 if (priv_dev->dev_ver <= DEV_VER_V2)
1999 writel(USB_CONF_DMULT, ®s->usb_conf);
2000
2001 if (priv_dev->dev_ver == DEV_VER_V2)
2002 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2);
2003
2004 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2005 u32 mask;
2006
2007 if (priv_ep->dir)
2008 mask = BIT(priv_ep->num + 16);
2009 else
2010 mask = BIT(priv_ep->num);
2011
2012 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
2013 cdns3_set_register_bit(®s->tdl_from_trb, mask);
2014 cdns3_set_register_bit(®s->tdl_beh, mask);
2015 cdns3_set_register_bit(®s->tdl_beh2, mask);
2016 cdns3_set_register_bit(®s->dma_adv_td, mask);
2017 }
2018
2019 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2020 cdns3_set_register_bit(®s->tdl_from_trb, mask);
2021
2022 cdns3_set_register_bit(®s->dtrans, mask);
2023 }
2024}
2025
2026/**
2027 * cdns3_ep_config - Configure hardware endpoint
2028 * @priv_ep: extended endpoint object
2029 * @enable: set EP_CFG_ENABLE bit in ep_cfg register.
2030 */
2031int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2032{
2033 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2034 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2035 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2036 u32 max_packet_size = 0;
2037 u8 maxburst = 0;
2038 u32 ep_cfg = 0;
2039 u8 buffering;
2040 u8 mult = 0;
2041 int ret;
2042
2043 buffering = priv_dev->ep_buf_size - 1;
2044
2045 cdns3_configure_dmult(priv_dev, priv_ep);
2046
2047 switch (priv_ep->type) {
2048 case USB_ENDPOINT_XFER_INT:
2049 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
2050
2051 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2052 ep_cfg |= EP_CFG_TDL_CHK;
2053 break;
2054 case USB_ENDPOINT_XFER_BULK:
2055 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
2056
2057 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2058 ep_cfg |= EP_CFG_TDL_CHK;
2059 break;
2060 default:
2061 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2062 mult = priv_dev->ep_iso_burst - 1;
2063 buffering = mult + 1;
2064 }
2065
2066 switch (priv_dev->gadget.speed) {
2067 case USB_SPEED_FULL:
2068 max_packet_size = is_iso_ep ? 1023 : 64;
2069 break;
2070 case USB_SPEED_HIGH:
2071 max_packet_size = is_iso_ep ? 1024 : 512;
2072 break;
2073 case USB_SPEED_SUPER:
2074 /* It's limitation that driver assumes in driver. */
2075 mult = 0;
2076 max_packet_size = 1024;
2077 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2078 maxburst = priv_dev->ep_iso_burst - 1;
2079 buffering = (mult + 1) *
2080 (maxburst + 1);
2081
2082 if (priv_ep->interval > 1)
2083 buffering++;
2084 } else {
2085 maxburst = priv_dev->ep_buf_size - 1;
2086 }
2087 break;
2088 default:
2089 /* all other speed are not supported */
2090 return -EINVAL;
2091 }
2092
2093 if (max_packet_size == 1024)
2094 priv_ep->trb_burst_size = 128;
2095 else if (max_packet_size >= 512)
2096 priv_ep->trb_burst_size = 64;
2097 else
2098 priv_ep->trb_burst_size = 16;
2099
2100 mult = min_t(u8, mult, EP_CFG_MULT_MAX);
2101 buffering = min_t(u8, buffering, EP_CFG_BUFFERING_MAX);
2102 maxburst = min_t(u8, maxburst, EP_CFG_MAXBURST_MAX);
2103
2104 /* onchip buffer is only allocated before configuration */
2105 if (!priv_dev->hw_configured_flag) {
2106 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2107 !!priv_ep->dir);
2108 if (ret) {
2109 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2110 return ret;
2111 }
2112 }
2113
2114 if (enable)
2115 ep_cfg |= EP_CFG_ENABLE;
2116
2117 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2118 if (priv_dev->dev_ver >= DEV_VER_V3) {
2119 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2120
2121 /*
2122 * Stream capable endpoints are handled by using ep_tdl
2123 * register. Other endpoints use TDL from TRB feature.
2124 */
2125 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,
2126 mask);
2127 }
2128
2129 /* Enable Stream Bit TDL chk and SID chk */
2130 ep_cfg |= EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK;
2131 }
2132
2133 ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2134 EP_CFG_MULT(mult) |
2135 EP_CFG_BUFFERING(buffering) |
2136 EP_CFG_MAXBURST(maxburst);
2137
2138 cdns3_select_ep(priv_dev, bEndpointAddress);
2139 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2140 priv_ep->flags |= EP_CONFIGURED;
2141
2142 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2143 priv_ep->name, ep_cfg);
2144
2145 return 0;
2146}
2147
2148/* Find correct direction for HW endpoint according to description */
2149static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2150 struct cdns3_endpoint *priv_ep)
2151{
2152 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2153 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2154}
2155
2156static struct
2157cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2158 struct usb_endpoint_descriptor *desc)
2159{
2160 struct usb_ep *ep;
2161 struct cdns3_endpoint *priv_ep;
2162
2163 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2164 unsigned long num;
2165 int ret;
2166 /* ep name pattern likes epXin or epXout */
2167 char c[2] = {ep->name[2], '\0'};
2168
2169 ret = kstrtoul(c, 10, &num);
2170 if (ret)
2171 return ERR_PTR(ret);
2172
2173 priv_ep = ep_to_cdns3_ep(ep);
2174 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2175 if (!(priv_ep->flags & EP_CLAIMED)) {
2176 priv_ep->num = num;
2177 return priv_ep;
2178 }
2179 }
2180 }
2181
2182 return ERR_PTR(-ENOENT);
2183}
2184
2185/*
2186 * Cadence IP has one limitation that all endpoints must be configured
2187 * (Type & MaxPacketSize) before setting configuration through hardware
2188 * register, it means we can't change endpoints configuration after
2189 * set_configuration.
2190 *
2191 * This function set EP_CLAIMED flag which is added when the gadget driver
2192 * uses usb_ep_autoconfig to configure specific endpoint;
2193 * When the udc driver receives set_configurion request,
2194 * it goes through all claimed endpoints, and configure all endpoints
2195 * accordingly.
2196 *
2197 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2198 * ep_cfg register which can be changed after set_configuration, and do
2199 * some software operation accordingly.
2200 */
2201static struct
2202usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2203 struct usb_endpoint_descriptor *desc,
2204 struct usb_ss_ep_comp_descriptor *comp_desc)
2205{
2206 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2207 struct cdns3_endpoint *priv_ep;
2208 unsigned long flags;
2209
2210 priv_ep = cdns3_find_available_ep(priv_dev, desc);
2211 if (IS_ERR(priv_ep)) {
2212 dev_err(priv_dev->dev, "no available ep\n");
2213 return NULL;
2214 }
2215
2216 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2217
2218 spin_lock_irqsave(&priv_dev->lock, flags);
2219 priv_ep->endpoint.desc = desc;
2220 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2221 priv_ep->type = usb_endpoint_type(desc);
2222 priv_ep->flags |= EP_CLAIMED;
2223 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2224
2225 spin_unlock_irqrestore(&priv_dev->lock, flags);
2226 return &priv_ep->endpoint;
2227}
2228
2229/**
2230 * cdns3_gadget_ep_alloc_request - Allocates request
2231 * @ep: endpoint object associated with request
2232 * @gfp_flags: gfp flags
2233 *
2234 * Returns allocated request address, NULL on allocation error
2235 */
2236struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2237 gfp_t gfp_flags)
2238{
2239 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2240 struct cdns3_request *priv_req;
2241
2242 priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2243 if (!priv_req)
2244 return NULL;
2245
2246 priv_req->priv_ep = priv_ep;
2247
2248 trace_cdns3_alloc_request(priv_req);
2249 return &priv_req->request;
2250}
2251
2252/**
2253 * cdns3_gadget_ep_free_request - Free memory occupied by request
2254 * @ep: endpoint object associated with request
2255 * @request: request to free memory
2256 */
2257void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2258 struct usb_request *request)
2259{
2260 struct cdns3_request *priv_req = to_cdns3_request(request);
2261
2262 if (priv_req->aligned_buf)
2263 priv_req->aligned_buf->in_use = 0;
2264
2265 trace_cdns3_free_request(priv_req);
2266 kfree(priv_req);
2267}
2268
2269/**
2270 * cdns3_gadget_ep_enable - Enable endpoint
2271 * @ep: endpoint object
2272 * @desc: endpoint descriptor
2273 *
2274 * Returns 0 on success, error code elsewhere
2275 */
2276static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2277 const struct usb_endpoint_descriptor *desc)
2278{
2279 struct cdns3_endpoint *priv_ep;
2280 struct cdns3_device *priv_dev;
2281 const struct usb_ss_ep_comp_descriptor *comp_desc;
2282 u32 reg = EP_STS_EN_TRBERREN;
2283 u32 bEndpointAddress;
2284 unsigned long flags;
2285 int enable = 1;
2286 int ret = 0;
2287 int val;
2288
2289 if (!ep) {
2290 pr_debug("usbss: ep not configured?\n");
2291 return -EINVAL;
2292 }
2293
2294 priv_ep = ep_to_cdns3_ep(ep);
2295 priv_dev = priv_ep->cdns3_dev;
2296 comp_desc = priv_ep->endpoint.comp_desc;
2297
2298 if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2299 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2300 return -EINVAL;
2301 }
2302
2303 if (!desc->wMaxPacketSize) {
2304 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2305 return -EINVAL;
2306 }
2307
2308 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2309 "%s is already enabled\n", priv_ep->name))
2310 return 0;
2311
2312 spin_lock_irqsave(&priv_dev->lock, flags);
2313
2314 priv_ep->endpoint.desc = desc;
2315 priv_ep->type = usb_endpoint_type(desc);
2316 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2317
2318 if (priv_ep->interval > ISO_MAX_INTERVAL &&
2319 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2320 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2321 ISO_MAX_INTERVAL);
2322
2323 ret = -EINVAL;
2324 goto exit;
2325 }
2326
2327 bEndpointAddress = priv_ep->num | priv_ep->dir;
2328 cdns3_select_ep(priv_dev, bEndpointAddress);
2329
2330 /*
2331 * For some versions of controller at some point during ISO OUT traffic
2332 * DMA reads Transfer Ring for the EP which has never got doorbell.
2333 * This issue was detected only on simulation, but to avoid this issue
2334 * driver add protection against it. To fix it driver enable ISO OUT
2335 * endpoint before setting DRBL. This special treatment of ISO OUT
2336 * endpoints are recommended by controller specification.
2337 */
2338 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2339 enable = 0;
2340
2341 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2342 /*
2343 * Enable stream support (SS mode) related interrupts
2344 * in EP_STS_EN Register
2345 */
2346 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2347 reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2348 EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2349 EP_STS_EN_STREAMREN;
2350 priv_ep->use_streams = true;
2351 ret = cdns3_ep_config(priv_ep, enable);
2352 priv_dev->using_streams |= true;
2353 }
2354 } else {
2355 ret = cdns3_ep_config(priv_ep, enable);
2356 }
2357
2358 if (ret)
2359 goto exit;
2360
2361 ret = cdns3_allocate_trb_pool(priv_ep);
2362 if (ret)
2363 goto exit;
2364
2365 bEndpointAddress = priv_ep->num | priv_ep->dir;
2366 cdns3_select_ep(priv_dev, bEndpointAddress);
2367
2368 trace_cdns3_gadget_ep_enable(priv_ep);
2369
2370 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2371
2372 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2373 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2374 1, 1000);
2375
2376 if (unlikely(ret)) {
2377 cdns3_free_trb_pool(priv_ep);
2378 ret = -EINVAL;
2379 goto exit;
2380 }
2381
2382 /* enable interrupt for selected endpoint */
2383 cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2384 BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2385
2386 if (priv_dev->dev_ver < DEV_VER_V2)
2387 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2388
2389 writel(reg, &priv_dev->regs->ep_sts_en);
2390
2391 ep->desc = desc;
2392 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2393 EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2394 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2395 priv_ep->wa1_set = 0;
2396 priv_ep->enqueue = 0;
2397 priv_ep->dequeue = 0;
2398 reg = readl(&priv_dev->regs->ep_sts);
2399 priv_ep->pcs = !!EP_STS_CCS(reg);
2400 priv_ep->ccs = !!EP_STS_CCS(reg);
2401 /* one TRB is reserved for link TRB used in DMULT mode*/
2402 priv_ep->free_trbs = priv_ep->num_trbs - 1;
2403exit:
2404 spin_unlock_irqrestore(&priv_dev->lock, flags);
2405
2406 return ret;
2407}
2408
2409/**
2410 * cdns3_gadget_ep_disable - Disable endpoint
2411 * @ep: endpoint object
2412 *
2413 * Returns 0 on success, error code elsewhere
2414 */
2415static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2416{
2417 struct cdns3_endpoint *priv_ep;
2418 struct cdns3_request *priv_req;
2419 struct cdns3_device *priv_dev;
2420 struct usb_request *request;
2421 unsigned long flags;
2422 int ret = 0;
2423 u32 ep_cfg;
2424 int val;
2425
2426 if (!ep) {
2427 pr_err("usbss: invalid parameters\n");
2428 return -EINVAL;
2429 }
2430
2431 priv_ep = ep_to_cdns3_ep(ep);
2432 priv_dev = priv_ep->cdns3_dev;
2433
2434 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2435 "%s is already disabled\n", priv_ep->name))
2436 return 0;
2437
2438 spin_lock_irqsave(&priv_dev->lock, flags);
2439
2440 trace_cdns3_gadget_ep_disable(priv_ep);
2441
2442 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2443
2444 ep_cfg = readl(&priv_dev->regs->ep_cfg);
2445 ep_cfg &= ~EP_CFG_ENABLE;
2446 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2447
2448 /**
2449 * Driver needs some time before resetting endpoint.
2450 * It need waits for clearing DBUSY bit or for timeout expired.
2451 * 10us is enough time for controller to stop transfer.
2452 */
2453 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2454 !(val & EP_STS_DBUSY), 1, 10);
2455 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2456
2457 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2458 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2459 1, 1000);
2460 if (unlikely(ret))
2461 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2462 priv_ep->name);
2463
2464 while (!list_empty(&priv_ep->pending_req_list)) {
2465 request = cdns3_next_request(&priv_ep->pending_req_list);
2466
2467 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2468 -ESHUTDOWN);
2469 }
2470
2471 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2472 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2473
2474 kfree(priv_req->request.buf);
2475 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2476 &priv_req->request);
2477 list_del_init(&priv_req->list);
2478 --priv_ep->wa2_counter;
2479 }
2480
2481 while (!list_empty(&priv_ep->deferred_req_list)) {
2482 request = cdns3_next_request(&priv_ep->deferred_req_list);
2483
2484 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2485 -ESHUTDOWN);
2486 }
2487
2488 priv_ep->descmis_req = NULL;
2489
2490 ep->desc = NULL;
2491 priv_ep->flags &= ~EP_ENABLED;
2492 priv_ep->use_streams = false;
2493
2494 spin_unlock_irqrestore(&priv_dev->lock, flags);
2495
2496 return ret;
2497}
2498
2499/**
2500 * __cdns3_gadget_ep_queue - Transfer data on endpoint
2501 * @ep: endpoint object
2502 * @request: request object
2503 * @gfp_flags: gfp flags
2504 *
2505 * Returns 0 on success, error code elsewhere
2506 */
2507static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2508 struct usb_request *request,
2509 gfp_t gfp_flags)
2510{
2511 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2512 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2513 struct cdns3_request *priv_req;
2514 int ret = 0;
2515
2516 request->actual = 0;
2517 request->status = -EINPROGRESS;
2518 priv_req = to_cdns3_request(request);
2519 trace_cdns3_ep_queue(priv_req);
2520
2521 if (priv_dev->dev_ver < DEV_VER_V2) {
2522 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2523 priv_req);
2524
2525 if (ret == EINPROGRESS)
2526 return 0;
2527 }
2528
2529 ret = cdns3_prepare_aligned_request_buf(priv_req);
2530 if (ret < 0)
2531 return ret;
2532
2533 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2534 usb_endpoint_dir_in(ep->desc));
2535 if (ret)
2536 return ret;
2537
2538 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2539
2540 /*
2541 * For stream capable endpoint if prime irq flag is set then only start
2542 * request.
2543 * If hardware endpoint configuration has not been set yet then
2544 * just queue request in deferred list. Transfer will be started in
2545 * cdns3_set_hw_configuration.
2546 */
2547 if (!request->stream_id) {
2548 if (priv_dev->hw_configured_flag &&
2549 !(priv_ep->flags & EP_STALLED) &&
2550 !(priv_ep->flags & EP_STALL_PENDING))
2551 cdns3_start_all_request(priv_dev, priv_ep);
2552 } else {
2553 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2554 cdns3_start_all_request(priv_dev, priv_ep);
2555 }
2556
2557 return 0;
2558}
2559
2560static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2561 gfp_t gfp_flags)
2562{
2563 struct usb_request *zlp_request;
2564 struct cdns3_endpoint *priv_ep;
2565 struct cdns3_device *priv_dev;
2566 unsigned long flags;
2567 int ret;
2568
2569 if (!request || !ep)
2570 return -EINVAL;
2571
2572 priv_ep = ep_to_cdns3_ep(ep);
2573 priv_dev = priv_ep->cdns3_dev;
2574
2575 spin_lock_irqsave(&priv_dev->lock, flags);
2576
2577 ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2578
2579 if (ret == 0 && request->zero && request->length &&
2580 (request->length % ep->maxpacket == 0)) {
2581 struct cdns3_request *priv_req;
2582
2583 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2584 zlp_request->buf = priv_dev->zlp_buf;
2585 zlp_request->length = 0;
2586
2587 priv_req = to_cdns3_request(zlp_request);
2588 priv_req->flags |= REQUEST_ZLP;
2589
2590 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2591 priv_ep->name);
2592 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2593 }
2594
2595 spin_unlock_irqrestore(&priv_dev->lock, flags);
2596 return ret;
2597}
2598
2599/**
2600 * cdns3_gadget_ep_dequeue - Remove request from transfer queue
2601 * @ep: endpoint object associated with request
2602 * @request: request object
2603 *
2604 * Returns 0 on success, error code elsewhere
2605 */
2606int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2607 struct usb_request *request)
2608{
2609 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2610 struct cdns3_device *priv_dev;
2611 struct usb_request *req, *req_temp;
2612 struct cdns3_request *priv_req;
2613 struct cdns3_trb *link_trb;
2614 u8 req_on_hw_ring = 0;
2615 unsigned long flags;
2616 int ret = 0;
2617 int val;
2618
2619 if (!ep || !request || !ep->desc)
2620 return -EINVAL;
2621
2622 priv_dev = priv_ep->cdns3_dev;
2623
2624 spin_lock_irqsave(&priv_dev->lock, flags);
2625
2626 priv_req = to_cdns3_request(request);
2627
2628 trace_cdns3_ep_dequeue(priv_req);
2629
2630 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2631
2632 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2633 list) {
2634 if (request == req) {
2635 req_on_hw_ring = 1;
2636 goto found;
2637 }
2638 }
2639
2640 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2641 list) {
2642 if (request == req)
2643 goto found;
2644 }
2645
2646 goto not_found;
2647
2648found:
2649 link_trb = priv_req->trb;
2650
2651 /* Update ring only if removed request is on pending_req_list list */
2652 if (req_on_hw_ring && link_trb) {
2653 /* Stop DMA */
2654 writel(EP_CMD_DFLUSH, &priv_dev->regs->ep_cmd);
2655
2656 /* wait for DFLUSH cleared */
2657 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2658 !(val & EP_CMD_DFLUSH), 1, 1000);
2659
2660 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2661 ((priv_req->end_trb + 1) * TRB_SIZE)));
2662 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
2663 TRB_TYPE(TRB_LINK) | TRB_CHAIN);
2664
2665 if (priv_ep->wa1_trb == priv_req->trb)
2666 cdns3_wa1_restore_cycle_bit(priv_ep);
2667 }
2668
2669 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2670
2671 req = cdns3_next_request(&priv_ep->pending_req_list);
2672 if (req)
2673 cdns3_rearm_transfer(priv_ep, 1);
2674
2675not_found:
2676 spin_unlock_irqrestore(&priv_dev->lock, flags);
2677 return ret;
2678}
2679
2680/**
2681 * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint
2682 * Should be called after acquiring spin_lock and selecting ep
2683 * @priv_ep: endpoint object to set stall on.
2684 */
2685void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2686{
2687 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2688
2689 trace_cdns3_halt(priv_ep, 1, 0);
2690
2691 if (!(priv_ep->flags & EP_STALLED)) {
2692 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2693
2694 if (!(ep_sts_reg & EP_STS_DBUSY))
2695 cdns3_ep_stall_flush(priv_ep);
2696 else
2697 priv_ep->flags |= EP_STALL_PENDING;
2698 }
2699}
2700
2701/**
2702 * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint
2703 * Should be called after acquiring spin_lock and selecting ep
2704 * @priv_ep: endpoint object to clear stall on
2705 */
2706int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2707{
2708 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2709 struct usb_request *request;
2710 struct cdns3_request *priv_req;
2711 struct cdns3_trb *trb = NULL;
2712 struct cdns3_trb trb_tmp;
2713 int ret;
2714 int val;
2715
2716 trace_cdns3_halt(priv_ep, 0, 0);
2717
2718 request = cdns3_next_request(&priv_ep->pending_req_list);
2719 if (request) {
2720 priv_req = to_cdns3_request(request);
2721 trb = priv_req->trb;
2722 if (trb) {
2723 trb_tmp = *trb;
2724 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2725 }
2726 }
2727
2728 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2729
2730 /* wait for EPRST cleared */
2731 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2732 !(val & EP_CMD_EPRST), 1, 100);
2733 if (ret)
2734 return -EINVAL;
2735
2736 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2737
2738 if (request) {
2739 if (trb)
2740 *trb = trb_tmp;
2741
2742 cdns3_rearm_transfer(priv_ep, 1);
2743 }
2744
2745 cdns3_start_all_request(priv_dev, priv_ep);
2746 return ret;
2747}
2748
2749/**
2750 * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint
2751 * @ep: endpoint object to set/clear stall on
2752 * @value: 1 for set stall, 0 for clear stall
2753 *
2754 * Returns 0 on success, error code elsewhere
2755 */
2756int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2757{
2758 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2759 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2760 unsigned long flags;
2761 int ret = 0;
2762
2763 if (!(priv_ep->flags & EP_ENABLED))
2764 return -EPERM;
2765
2766 spin_lock_irqsave(&priv_dev->lock, flags);
2767
2768 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2769
2770 if (!value) {
2771 priv_ep->flags &= ~EP_WEDGE;
2772 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2773 } else {
2774 __cdns3_gadget_ep_set_halt(priv_ep);
2775 }
2776
2777 spin_unlock_irqrestore(&priv_dev->lock, flags);
2778
2779 return ret;
2780}
2781
2782extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2783
2784static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2785 .enable = cdns3_gadget_ep_enable,
2786 .disable = cdns3_gadget_ep_disable,
2787 .alloc_request = cdns3_gadget_ep_alloc_request,
2788 .free_request = cdns3_gadget_ep_free_request,
2789 .queue = cdns3_gadget_ep_queue,
2790 .dequeue = cdns3_gadget_ep_dequeue,
2791 .set_halt = cdns3_gadget_ep_set_halt,
2792 .set_wedge = cdns3_gadget_ep_set_wedge,
2793};
2794
2795/**
2796 * cdns3_gadget_get_frame - Returns number of actual ITP frame
2797 * @gadget: gadget object
2798 *
2799 * Returns number of actual ITP frame
2800 */
2801static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2802{
2803 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2804
2805 return readl(&priv_dev->regs->usb_itpn);
2806}
2807
2808int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2809{
2810 enum usb_device_speed speed;
2811
2812 speed = cdns3_get_speed(priv_dev);
2813
2814 if (speed >= USB_SPEED_SUPER)
2815 return 0;
2816
2817 /* Start driving resume signaling to indicate remote wakeup. */
2818 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2819
2820 return 0;
2821}
2822
2823static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2824{
2825 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2826 unsigned long flags;
2827 int ret = 0;
2828
2829 spin_lock_irqsave(&priv_dev->lock, flags);
2830 ret = __cdns3_gadget_wakeup(priv_dev);
2831 spin_unlock_irqrestore(&priv_dev->lock, flags);
2832 return ret;
2833}
2834
2835static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2836 int is_selfpowered)
2837{
2838 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2839 unsigned long flags;
2840
2841 spin_lock_irqsave(&priv_dev->lock, flags);
2842 priv_dev->is_selfpowered = !!is_selfpowered;
2843 spin_unlock_irqrestore(&priv_dev->lock, flags);
2844 return 0;
2845}
2846
2847static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2848{
2849 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2850
2851 if (is_on) {
2852 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2853 } else {
2854 writel(~0, &priv_dev->regs->ep_ists);
2855 writel(~0, &priv_dev->regs->usb_ists);
2856 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2857 }
2858
2859 return 0;
2860}
2861
2862static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2863{
2864 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2865 u32 reg;
2866
2867 cdns3_ep0_config(priv_dev);
2868
2869 /* enable interrupts for endpoint 0 (in and out) */
2870 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien);
2871
2872 /*
2873 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2874 * revision of controller.
2875 */
2876 if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2877 reg = readl(®s->dbg_link1);
2878
2879 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2880 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2881 DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2882 writel(reg, ®s->dbg_link1);
2883 }
2884
2885 /*
2886 * By default some platforms has set protected access to memory.
2887 * This cause problem with cache, so driver restore non-secure
2888 * access to memory.
2889 */
2890 reg = readl(®s->dma_axi_ctrl);
2891 reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2892 DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2893 writel(reg, ®s->dma_axi_ctrl);
2894
2895 /* enable generic interrupt*/
2896 writel(USB_IEN_INIT, ®s->usb_ien);
2897 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf);
2898 /* keep Fast Access bit */
2899 writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);
2900
2901 cdns3_configure_dmult(priv_dev, NULL);
2902}
2903
2904/**
2905 * cdns3_gadget_udc_start - Gadget start
2906 * @gadget: gadget object
2907 * @driver: driver which operates on this gadget
2908 *
2909 * Returns 0 on success, error code elsewhere
2910 */
2911static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2912 struct usb_gadget_driver *driver)
2913{
2914 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2915 unsigned long flags;
2916 enum usb_device_speed max_speed = driver->max_speed;
2917
2918 spin_lock_irqsave(&priv_dev->lock, flags);
2919 priv_dev->gadget_driver = driver;
2920
2921 /* limit speed if necessary */
2922 max_speed = min(driver->max_speed, gadget->max_speed);
2923
2924 switch (max_speed) {
2925 case USB_SPEED_FULL:
2926 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2927 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2928 break;
2929 case USB_SPEED_HIGH:
2930 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2931 break;
2932 case USB_SPEED_SUPER:
2933 break;
2934 default:
2935 dev_err(priv_dev->dev,
2936 "invalid maximum_speed parameter %d\n",
2937 max_speed);
2938 fallthrough;
2939 case USB_SPEED_UNKNOWN:
2940 /* default to superspeed */
2941 max_speed = USB_SPEED_SUPER;
2942 break;
2943 }
2944
2945 cdns3_gadget_config(priv_dev);
2946 spin_unlock_irqrestore(&priv_dev->lock, flags);
2947 return 0;
2948}
2949
2950/**
2951 * cdns3_gadget_udc_stop - Stops gadget
2952 * @gadget: gadget object
2953 *
2954 * Returns 0
2955 */
2956static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2957{
2958 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2959 struct cdns3_endpoint *priv_ep;
2960 u32 bEndpointAddress;
2961 struct usb_ep *ep;
2962 int val;
2963
2964 priv_dev->gadget_driver = NULL;
2965
2966 priv_dev->onchip_used_size = 0;
2967 priv_dev->out_mem_is_allocated = 0;
2968 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2969
2970 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2971 priv_ep = ep_to_cdns3_ep(ep);
2972 bEndpointAddress = priv_ep->num | priv_ep->dir;
2973 cdns3_select_ep(priv_dev, bEndpointAddress);
2974 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2975 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2976 !(val & EP_CMD_EPRST), 1, 100);
2977
2978 priv_ep->flags &= ~EP_CLAIMED;
2979 }
2980
2981 /* disable interrupt for device */
2982 writel(0, &priv_dev->regs->usb_ien);
2983 writel(0, &priv_dev->regs->usb_pwr);
2984 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2985
2986 return 0;
2987}
2988
2989/**
2990 * cdns3_gadget_check_config - ensure cdns3 can support the USB configuration
2991 * @gadget: pointer to the USB gadget
2992 *
2993 * Used to record the maximum number of endpoints being used in a USB composite
2994 * device. (across all configurations) This is to be used in the calculation
2995 * of the TXFIFO sizes when resizing internal memory for individual endpoints.
2996 * It will help ensured that the resizing logic reserves enough space for at
2997 * least one max packet.
2998 */
2999static int cdns3_gadget_check_config(struct usb_gadget *gadget)
3000{
3001 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
3002 struct usb_ep *ep;
3003 int n_in = 0;
3004 int total;
3005
3006 list_for_each_entry(ep, &gadget->ep_list, ep_list) {
3007 if (ep->claimed && (ep->address & USB_DIR_IN))
3008 n_in++;
3009 }
3010
3011 /* 2KB are reserved for EP0, 1KB for out*/
3012 total = 2 + n_in + 1;
3013
3014 if (total > priv_dev->onchip_buffers)
3015 return -ENOMEM;
3016
3017 priv_dev->ep_buf_size = priv_dev->ep_iso_burst =
3018 (priv_dev->onchip_buffers - 2) / (n_in + 1);
3019
3020 return 0;
3021}
3022
3023static const struct usb_gadget_ops cdns3_gadget_ops = {
3024 .get_frame = cdns3_gadget_get_frame,
3025 .wakeup = cdns3_gadget_wakeup,
3026 .set_selfpowered = cdns3_gadget_set_selfpowered,
3027 .pullup = cdns3_gadget_pullup,
3028 .udc_start = cdns3_gadget_udc_start,
3029 .udc_stop = cdns3_gadget_udc_stop,
3030 .match_ep = cdns3_gadget_match_ep,
3031 .check_config = cdns3_gadget_check_config,
3032};
3033
3034static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
3035{
3036 int i;
3037
3038 /* ep0 OUT point to ep0 IN. */
3039 priv_dev->eps[16] = NULL;
3040
3041 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
3042 if (priv_dev->eps[i]) {
3043 cdns3_free_trb_pool(priv_dev->eps[i]);
3044 devm_kfree(priv_dev->dev, priv_dev->eps[i]);
3045 }
3046}
3047
3048/**
3049 * cdns3_init_eps - Initializes software endpoints of gadget
3050 * @priv_dev: extended gadget object
3051 *
3052 * Returns 0 on success, error code elsewhere
3053 */
3054static int cdns3_init_eps(struct cdns3_device *priv_dev)
3055{
3056 u32 ep_enabled_reg, iso_ep_reg;
3057 struct cdns3_endpoint *priv_ep;
3058 int ep_dir, ep_number;
3059 u32 ep_mask;
3060 int ret = 0;
3061 int i;
3062
3063 /* Read it from USB_CAP3 to USB_CAP5 */
3064 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
3065 iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
3066
3067 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
3068
3069 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
3070 ep_dir = i >> 4; /* i div 16 */
3071 ep_number = i & 0xF; /* i % 16 */
3072 ep_mask = BIT(i);
3073
3074 if (!(ep_enabled_reg & ep_mask))
3075 continue;
3076
3077 if (ep_dir && !ep_number) {
3078 priv_dev->eps[i] = priv_dev->eps[0];
3079 continue;
3080 }
3081
3082 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3083 GFP_KERNEL);
3084 if (!priv_ep)
3085 goto err;
3086
3087 /* set parent of endpoint object */
3088 priv_ep->cdns3_dev = priv_dev;
3089 priv_dev->eps[i] = priv_ep;
3090 priv_ep->num = ep_number;
3091 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3092
3093 if (!ep_number) {
3094 ret = cdns3_init_ep0(priv_dev, priv_ep);
3095 if (ret) {
3096 dev_err(priv_dev->dev, "Failed to init ep0\n");
3097 goto err;
3098 }
3099 } else {
3100 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3101 ep_number, !!ep_dir ? "in" : "out");
3102 priv_ep->endpoint.name = priv_ep->name;
3103
3104 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3105 CDNS3_EP_MAX_PACKET_LIMIT);
3106 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3107 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3108 if (ep_dir)
3109 priv_ep->endpoint.caps.dir_in = 1;
3110 else
3111 priv_ep->endpoint.caps.dir_out = 1;
3112
3113 if (iso_ep_reg & ep_mask)
3114 priv_ep->endpoint.caps.type_iso = 1;
3115
3116 priv_ep->endpoint.caps.type_bulk = 1;
3117 priv_ep->endpoint.caps.type_int = 1;
3118
3119 list_add_tail(&priv_ep->endpoint.ep_list,
3120 &priv_dev->gadget.ep_list);
3121 }
3122
3123 priv_ep->flags = 0;
3124
3125 dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n",
3126 priv_ep->name,
3127 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3128 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3129
3130 INIT_LIST_HEAD(&priv_ep->pending_req_list);
3131 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3132 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
3133 }
3134
3135 return 0;
3136err:
3137 cdns3_free_all_eps(priv_dev);
3138 return -ENOMEM;
3139}
3140
3141static void cdns3_gadget_release(struct device *dev)
3142{
3143 struct cdns3_device *priv_dev = container_of(dev,
3144 struct cdns3_device, gadget.dev);
3145
3146 kfree(priv_dev);
3147}
3148
3149static void cdns3_gadget_exit(struct cdns *cdns)
3150{
3151 struct cdns3_device *priv_dev;
3152
3153 priv_dev = cdns->gadget_dev;
3154
3155
3156 pm_runtime_mark_last_busy(cdns->dev);
3157 pm_runtime_put_autosuspend(cdns->dev);
3158
3159 usb_del_gadget(&priv_dev->gadget);
3160 devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
3161
3162 cdns3_free_all_eps(priv_dev);
3163
3164 while (!list_empty(&priv_dev->aligned_buf_list)) {
3165 struct cdns3_aligned_buf *buf;
3166
3167 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3168 dma_free_noncoherent(priv_dev->sysdev, buf->size,
3169 buf->buf,
3170 buf->dma,
3171 buf->dir);
3172
3173 list_del(&buf->list);
3174 kfree(buf);
3175 }
3176
3177 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3178 priv_dev->setup_dma);
3179 dma_pool_destroy(priv_dev->eps_dma_pool);
3180
3181 kfree(priv_dev->zlp_buf);
3182 usb_put_gadget(&priv_dev->gadget);
3183 cdns->gadget_dev = NULL;
3184 cdns_drd_gadget_off(cdns);
3185}
3186
3187static int cdns3_gadget_start(struct cdns *cdns)
3188{
3189 struct cdns3_device *priv_dev;
3190 u32 max_speed;
3191 int ret;
3192
3193 priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3194 if (!priv_dev)
3195 return -ENOMEM;
3196
3197 usb_initialize_gadget(cdns->dev, &priv_dev->gadget,
3198 cdns3_gadget_release);
3199 cdns->gadget_dev = priv_dev;
3200 priv_dev->sysdev = cdns->dev;
3201 priv_dev->dev = cdns->dev;
3202 priv_dev->regs = cdns->dev_regs;
3203
3204 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3205 &priv_dev->onchip_buffers);
3206
3207 if (priv_dev->onchip_buffers <= 0) {
3208 u32 reg = readl(&priv_dev->regs->usb_cap2);
3209
3210 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3211 }
3212
3213 if (!priv_dev->onchip_buffers)
3214 priv_dev->onchip_buffers = 256;
3215
3216 max_speed = usb_get_maximum_speed(cdns->dev);
3217
3218 /* Check the maximum_speed parameter */
3219 switch (max_speed) {
3220 case USB_SPEED_FULL:
3221 case USB_SPEED_HIGH:
3222 case USB_SPEED_SUPER:
3223 break;
3224 default:
3225 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3226 max_speed);
3227 fallthrough;
3228 case USB_SPEED_UNKNOWN:
3229 /* default to superspeed */
3230 max_speed = USB_SPEED_SUPER;
3231 break;
3232 }
3233
3234 /* fill gadget fields */
3235 priv_dev->gadget.max_speed = max_speed;
3236 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3237 priv_dev->gadget.ops = &cdns3_gadget_ops;
3238 priv_dev->gadget.name = "usb-ss-gadget";
3239 priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3240 priv_dev->gadget.irq = cdns->dev_irq;
3241
3242 spin_lock_init(&priv_dev->lock);
3243 INIT_WORK(&priv_dev->pending_status_wq,
3244 cdns3_pending_setup_status_handler);
3245
3246 INIT_WORK(&priv_dev->aligned_buf_wq,
3247 cdns3_free_aligned_request_buf);
3248
3249 /* initialize endpoint container */
3250 INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3251 INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3252 priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool",
3253 priv_dev->sysdev,
3254 TRB_RING_SIZE, 8, 0);
3255 if (!priv_dev->eps_dma_pool) {
3256 dev_err(priv_dev->dev, "Failed to create TRB dma pool\n");
3257 ret = -ENOMEM;
3258 goto err1;
3259 }
3260
3261 ret = cdns3_init_eps(priv_dev);
3262 if (ret) {
3263 dev_err(priv_dev->dev, "Failed to create endpoints\n");
3264 goto err1;
3265 }
3266
3267 /* allocate memory for setup packet buffer */
3268 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3269 &priv_dev->setup_dma, GFP_DMA);
3270 if (!priv_dev->setup_buf) {
3271 ret = -ENOMEM;
3272 goto err2;
3273 }
3274
3275 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3276
3277 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3278 readl(&priv_dev->regs->usb_cap6));
3279 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3280 readl(&priv_dev->regs->usb_cap1));
3281 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3282 readl(&priv_dev->regs->usb_cap2));
3283
3284 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3285 if (priv_dev->dev_ver >= DEV_VER_V2)
3286 priv_dev->gadget.sg_supported = 1;
3287
3288 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3289 if (!priv_dev->zlp_buf) {
3290 ret = -ENOMEM;
3291 goto err3;
3292 }
3293
3294 /* add USB gadget device */
3295 ret = usb_add_gadget(&priv_dev->gadget);
3296 if (ret < 0) {
3297 dev_err(priv_dev->dev, "Failed to add gadget\n");
3298 goto err4;
3299 }
3300
3301 return 0;
3302err4:
3303 kfree(priv_dev->zlp_buf);
3304err3:
3305 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3306 priv_dev->setup_dma);
3307err2:
3308 cdns3_free_all_eps(priv_dev);
3309err1:
3310 dma_pool_destroy(priv_dev->eps_dma_pool);
3311
3312 usb_put_gadget(&priv_dev->gadget);
3313 cdns->gadget_dev = NULL;
3314 return ret;
3315}
3316
3317static int __cdns3_gadget_init(struct cdns *cdns)
3318{
3319 int ret = 0;
3320
3321 /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3322 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3323 if (ret) {
3324 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3325 return ret;
3326 }
3327
3328 cdns_drd_gadget_on(cdns);
3329 pm_runtime_get_sync(cdns->dev);
3330
3331 ret = cdns3_gadget_start(cdns);
3332 if (ret) {
3333 pm_runtime_put_sync(cdns->dev);
3334 return ret;
3335 }
3336
3337 /*
3338 * Because interrupt line can be shared with other components in
3339 * driver it can't use IRQF_ONESHOT flag here.
3340 */
3341 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3342 cdns3_device_irq_handler,
3343 cdns3_device_thread_irq_handler,
3344 IRQF_SHARED, dev_name(cdns->dev),
3345 cdns->gadget_dev);
3346
3347 if (ret)
3348 goto err0;
3349
3350 return 0;
3351err0:
3352 cdns3_gadget_exit(cdns);
3353 return ret;
3354}
3355
3356static int cdns3_gadget_suspend(struct cdns *cdns, bool do_wakeup)
3357__must_hold(&cdns->lock)
3358{
3359 struct cdns3_device *priv_dev = cdns->gadget_dev;
3360
3361 spin_unlock(&cdns->lock);
3362 cdns3_disconnect_gadget(priv_dev);
3363 spin_lock(&cdns->lock);
3364
3365 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3366 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3367 cdns3_hw_reset_eps_config(priv_dev);
3368
3369 /* disable interrupt for device */
3370 writel(0, &priv_dev->regs->usb_ien);
3371
3372 return 0;
3373}
3374
3375static int cdns3_gadget_resume(struct cdns *cdns, bool hibernated)
3376{
3377 struct cdns3_device *priv_dev = cdns->gadget_dev;
3378
3379 if (!priv_dev->gadget_driver)
3380 return 0;
3381
3382 cdns3_gadget_config(priv_dev);
3383 if (hibernated)
3384 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
3385
3386 return 0;
3387}
3388
3389/**
3390 * cdns3_gadget_init - initialize device structure
3391 *
3392 * @cdns: cdns instance
3393 *
3394 * This function initializes the gadget.
3395 */
3396int cdns3_gadget_init(struct cdns *cdns)
3397{
3398 struct cdns_role_driver *rdrv;
3399
3400 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3401 if (!rdrv)
3402 return -ENOMEM;
3403
3404 rdrv->start = __cdns3_gadget_init;
3405 rdrv->stop = cdns3_gadget_exit;
3406 rdrv->suspend = cdns3_gadget_suspend;
3407 rdrv->resume = cdns3_gadget_resume;
3408 rdrv->state = CDNS_ROLE_STATE_INACTIVE;
3409 rdrv->name = "gadget";
3410 cdns->roles[USB_ROLE_DEVICE] = rdrv;
3411
3412 return 0;
3413}