Loading...
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * NVIDIA Tegra XUSB device mode controller
4 *
5 * Copyright (c) 2013-2022, NVIDIA CORPORATION. All rights reserved.
6 * Copyright (c) 2015, Google Inc.
7 */
8
9#include <linux/clk.h>
10#include <linux/completion.h>
11#include <linux/delay.h>
12#include <linux/dma-mapping.h>
13#include <linux/dmapool.h>
14#include <linux/interrupt.h>
15#include <linux/iopoll.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/of.h>
19#include <linux/phy/phy.h>
20#include <linux/phy/tegra/xusb.h>
21#include <linux/pm_domain.h>
22#include <linux/platform_device.h>
23#include <linux/pm_runtime.h>
24#include <linux/regulator/consumer.h>
25#include <linux/reset.h>
26#include <linux/usb/ch9.h>
27#include <linux/usb/gadget.h>
28#include <linux/usb/otg.h>
29#include <linux/usb/role.h>
30#include <linux/usb/phy.h>
31#include <linux/workqueue.h>
32
33/* XUSB_DEV registers */
34#define DB 0x004
35#define DB_TARGET_MASK GENMASK(15, 8)
36#define DB_TARGET(x) (((x) << 8) & DB_TARGET_MASK)
37#define DB_STREAMID_MASK GENMASK(31, 16)
38#define DB_STREAMID(x) (((x) << 16) & DB_STREAMID_MASK)
39#define ERSTSZ 0x008
40#define ERSTSZ_ERSTXSZ_SHIFT(x) ((x) * 16)
41#define ERSTSZ_ERSTXSZ_MASK GENMASK(15, 0)
42#define ERSTXBALO(x) (0x010 + 8 * (x))
43#define ERSTXBAHI(x) (0x014 + 8 * (x))
44#define ERDPLO 0x020
45#define ERDPLO_EHB BIT(3)
46#define ERDPHI 0x024
47#define EREPLO 0x028
48#define EREPLO_ECS BIT(0)
49#define EREPLO_SEGI BIT(1)
50#define EREPHI 0x02c
51#define CTRL 0x030
52#define CTRL_RUN BIT(0)
53#define CTRL_LSE BIT(1)
54#define CTRL_IE BIT(4)
55#define CTRL_SMI_EVT BIT(5)
56#define CTRL_SMI_DSE BIT(6)
57#define CTRL_EWE BIT(7)
58#define CTRL_DEVADDR_MASK GENMASK(30, 24)
59#define CTRL_DEVADDR(x) (((x) << 24) & CTRL_DEVADDR_MASK)
60#define CTRL_ENABLE BIT(31)
61#define ST 0x034
62#define ST_RC BIT(0)
63#define ST_IP BIT(4)
64#define RT_IMOD 0x038
65#define RT_IMOD_IMODI_MASK GENMASK(15, 0)
66#define RT_IMOD_IMODI(x) ((x) & RT_IMOD_IMODI_MASK)
67#define RT_IMOD_IMODC_MASK GENMASK(31, 16)
68#define RT_IMOD_IMODC(x) (((x) << 16) & RT_IMOD_IMODC_MASK)
69#define PORTSC 0x03c
70#define PORTSC_CCS BIT(0)
71#define PORTSC_PED BIT(1)
72#define PORTSC_PR BIT(4)
73#define PORTSC_PLS_SHIFT 5
74#define PORTSC_PLS_MASK GENMASK(8, 5)
75#define PORTSC_PLS_U0 0x0
76#define PORTSC_PLS_U2 0x2
77#define PORTSC_PLS_U3 0x3
78#define PORTSC_PLS_DISABLED 0x4
79#define PORTSC_PLS_RXDETECT 0x5
80#define PORTSC_PLS_INACTIVE 0x6
81#define PORTSC_PLS_RESUME 0xf
82#define PORTSC_PLS(x) (((x) << PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK)
83#define PORTSC_PS_SHIFT 10
84#define PORTSC_PS_MASK GENMASK(13, 10)
85#define PORTSC_PS_UNDEFINED 0x0
86#define PORTSC_PS_FS 0x1
87#define PORTSC_PS_LS 0x2
88#define PORTSC_PS_HS 0x3
89#define PORTSC_PS_SS 0x4
90#define PORTSC_LWS BIT(16)
91#define PORTSC_CSC BIT(17)
92#define PORTSC_WRC BIT(19)
93#define PORTSC_PRC BIT(21)
94#define PORTSC_PLC BIT(22)
95#define PORTSC_CEC BIT(23)
96#define PORTSC_WPR BIT(30)
97#define PORTSC_CHANGE_MASK (PORTSC_CSC | PORTSC_WRC | PORTSC_PRC | \
98 PORTSC_PLC | PORTSC_CEC)
99#define ECPLO 0x040
100#define ECPHI 0x044
101#define MFINDEX 0x048
102#define MFINDEX_FRAME_SHIFT 3
103#define MFINDEX_FRAME_MASK GENMASK(13, 3)
104#define PORTPM 0x04c
105#define PORTPM_L1S_MASK GENMASK(1, 0)
106#define PORTPM_L1S_DROP 0x0
107#define PORTPM_L1S_ACCEPT 0x1
108#define PORTPM_L1S_NYET 0x2
109#define PORTPM_L1S_STALL 0x3
110#define PORTPM_L1S(x) ((x) & PORTPM_L1S_MASK)
111#define PORTPM_RWE BIT(3)
112#define PORTPM_U2TIMEOUT_MASK GENMASK(15, 8)
113#define PORTPM_U1TIMEOUT_MASK GENMASK(23, 16)
114#define PORTPM_FLA BIT(24)
115#define PORTPM_VBA BIT(25)
116#define PORTPM_WOC BIT(26)
117#define PORTPM_WOD BIT(27)
118#define PORTPM_U1E BIT(28)
119#define PORTPM_U2E BIT(29)
120#define PORTPM_FRWE BIT(30)
121#define PORTPM_PNG_CYA BIT(31)
122#define EP_HALT 0x050
123#define EP_PAUSE 0x054
124#define EP_RELOAD 0x058
125#define EP_STCHG 0x05c
126#define DEVNOTIF_LO 0x064
127#define DEVNOTIF_LO_TRIG BIT(0)
128#define DEVNOTIF_LO_TYPE_MASK GENMASK(7, 4)
129#define DEVNOTIF_LO_TYPE(x) (((x) << 4) & DEVNOTIF_LO_TYPE_MASK)
130#define DEVNOTIF_LO_TYPE_FUNCTION_WAKE 0x1
131#define DEVNOTIF_HI 0x068
132#define PORTHALT 0x06c
133#define PORTHALT_HALT_LTSSM BIT(0)
134#define PORTHALT_HALT_REJECT BIT(1)
135#define PORTHALT_STCHG_REQ BIT(20)
136#define PORTHALT_STCHG_INTR_EN BIT(24)
137#define PORT_TM 0x070
138#define EP_THREAD_ACTIVE 0x074
139#define EP_STOPPED 0x078
140#define HSFSPI_COUNT0 0x100
141#define HSFSPI_COUNT13 0x134
142#define HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK GENMASK(29, 0)
143#define HSFSPI_COUNT13_U2_RESUME_K_DURATION(x) ((x) & \
144 HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK)
145#define BLCG 0x840
146#define SSPX_CORE_CNT0 0x610
147#define SSPX_CORE_CNT0_PING_TBURST_MASK GENMASK(7, 0)
148#define SSPX_CORE_CNT0_PING_TBURST(x) ((x) & SSPX_CORE_CNT0_PING_TBURST_MASK)
149#define SSPX_CORE_CNT30 0x688
150#define SSPX_CORE_CNT30_LMPITP_TIMER_MASK GENMASK(19, 0)
151#define SSPX_CORE_CNT30_LMPITP_TIMER(x) ((x) & \
152 SSPX_CORE_CNT30_LMPITP_TIMER_MASK)
153#define SSPX_CORE_CNT32 0x690
154#define SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK GENMASK(7, 0)
155#define SSPX_CORE_CNT32_POLL_TBURST_MAX(x) ((x) & \
156 SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK)
157#define SSPX_CORE_CNT56 0x6fc
158#define SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK GENMASK(19, 0)
159#define SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(x) ((x) & \
160 SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK)
161#define SSPX_CORE_CNT57 0x700
162#define SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK GENMASK(19, 0)
163#define SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(x) ((x) & \
164 SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK)
165#define SSPX_CORE_CNT65 0x720
166#define SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK GENMASK(19, 0)
167#define SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(x) ((x) & \
168 SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK)
169#define SSPX_CORE_CNT66 0x724
170#define SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK GENMASK(19, 0)
171#define SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(x) ((x) & \
172 SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK)
173#define SSPX_CORE_CNT67 0x728
174#define SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK GENMASK(19, 0)
175#define SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(x) ((x) & \
176 SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK)
177#define SSPX_CORE_CNT72 0x73c
178#define SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK GENMASK(19, 0)
179#define SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(x) ((x) & \
180 SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK)
181#define SSPX_CORE_PADCTL4 0x750
182#define SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK GENMASK(19, 0)
183#define SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(x) ((x) & \
184 SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK)
185#define BLCG_DFPCI BIT(0)
186#define BLCG_UFPCI BIT(1)
187#define BLCG_FE BIT(2)
188#define BLCG_COREPLL_PWRDN BIT(8)
189#define BLCG_IOPLL_0_PWRDN BIT(9)
190#define BLCG_IOPLL_1_PWRDN BIT(10)
191#define BLCG_IOPLL_2_PWRDN BIT(11)
192#define BLCG_ALL 0x1ff
193#define CFG_DEV_SSPI_XFER 0x858
194#define CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK GENMASK(31, 0)
195#define CFG_DEV_SSPI_XFER_ACKTIMEOUT(x) ((x) & \
196 CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK)
197#define CFG_DEV_FE 0x85c
198#define CFG_DEV_FE_PORTREGSEL_MASK GENMASK(1, 0)
199#define CFG_DEV_FE_PORTREGSEL_SS_PI 1
200#define CFG_DEV_FE_PORTREGSEL_HSFS_PI 2
201#define CFG_DEV_FE_PORTREGSEL(x) ((x) & CFG_DEV_FE_PORTREGSEL_MASK)
202#define CFG_DEV_FE_INFINITE_SS_RETRY BIT(29)
203
204/* FPCI registers */
205#define XUSB_DEV_CFG_1 0x004
206#define XUSB_DEV_CFG_1_IO_SPACE_EN BIT(0)
207#define XUSB_DEV_CFG_1_MEMORY_SPACE_EN BIT(1)
208#define XUSB_DEV_CFG_1_BUS_MASTER_EN BIT(2)
209#define XUSB_DEV_CFG_4 0x010
210#define XUSB_DEV_CFG_4_BASE_ADDR_MASK GENMASK(31, 15)
211#define XUSB_DEV_CFG_5 0x014
212
213/* IPFS registers */
214#define XUSB_DEV_CONFIGURATION_0 0x180
215#define XUSB_DEV_CONFIGURATION_0_EN_FPCI BIT(0)
216#define XUSB_DEV_INTR_MASK_0 0x188
217#define XUSB_DEV_INTR_MASK_0_IP_INT_MASK BIT(16)
218
219struct tegra_xudc_ep_context {
220 __le32 info0;
221 __le32 info1;
222 __le32 deq_lo;
223 __le32 deq_hi;
224 __le32 tx_info;
225 __le32 rsvd[11];
226};
227
228#define EP_STATE_DISABLED 0
229#define EP_STATE_RUNNING 1
230#define EP_STATE_HALTED 2
231#define EP_STATE_STOPPED 3
232#define EP_STATE_ERROR 4
233
234#define EP_TYPE_INVALID 0
235#define EP_TYPE_ISOCH_OUT 1
236#define EP_TYPE_BULK_OUT 2
237#define EP_TYPE_INTERRUPT_OUT 3
238#define EP_TYPE_CONTROL 4
239#define EP_TYPE_ISCOH_IN 5
240#define EP_TYPE_BULK_IN 6
241#define EP_TYPE_INTERRUPT_IN 7
242
243#define BUILD_EP_CONTEXT_RW(name, member, shift, mask) \
244static inline u32 ep_ctx_read_##name(struct tegra_xudc_ep_context *ctx) \
245{ \
246 return (le32_to_cpu(ctx->member) >> (shift)) & (mask); \
247} \
248static inline void \
249ep_ctx_write_##name(struct tegra_xudc_ep_context *ctx, u32 val) \
250{ \
251 u32 tmp; \
252 \
253 tmp = le32_to_cpu(ctx->member) & ~((mask) << (shift)); \
254 tmp |= (val & (mask)) << (shift); \
255 ctx->member = cpu_to_le32(tmp); \
256}
257
258BUILD_EP_CONTEXT_RW(state, info0, 0, 0x7)
259BUILD_EP_CONTEXT_RW(mult, info0, 8, 0x3)
260BUILD_EP_CONTEXT_RW(max_pstreams, info0, 10, 0x1f)
261BUILD_EP_CONTEXT_RW(lsa, info0, 15, 0x1)
262BUILD_EP_CONTEXT_RW(interval, info0, 16, 0xff)
263BUILD_EP_CONTEXT_RW(cerr, info1, 1, 0x3)
264BUILD_EP_CONTEXT_RW(type, info1, 3, 0x7)
265BUILD_EP_CONTEXT_RW(hid, info1, 7, 0x1)
266BUILD_EP_CONTEXT_RW(max_burst_size, info1, 8, 0xff)
267BUILD_EP_CONTEXT_RW(max_packet_size, info1, 16, 0xffff)
268BUILD_EP_CONTEXT_RW(dcs, deq_lo, 0, 0x1)
269BUILD_EP_CONTEXT_RW(deq_lo, deq_lo, 4, 0xfffffff)
270BUILD_EP_CONTEXT_RW(deq_hi, deq_hi, 0, 0xffffffff)
271BUILD_EP_CONTEXT_RW(avg_trb_len, tx_info, 0, 0xffff)
272BUILD_EP_CONTEXT_RW(max_esit_payload, tx_info, 16, 0xffff)
273BUILD_EP_CONTEXT_RW(edtla, rsvd[0], 0, 0xffffff)
274BUILD_EP_CONTEXT_RW(rsvd, rsvd[0], 24, 0x1)
275BUILD_EP_CONTEXT_RW(partial_td, rsvd[0], 25, 0x1)
276BUILD_EP_CONTEXT_RW(splitxstate, rsvd[0], 26, 0x1)
277BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 27, 0x1f)
278BUILD_EP_CONTEXT_RW(cerrcnt, rsvd[1], 18, 0x3)
279BUILD_EP_CONTEXT_RW(data_offset, rsvd[2], 0, 0x1ffff)
280BUILD_EP_CONTEXT_RW(numtrbs, rsvd[2], 22, 0x1f)
281BUILD_EP_CONTEXT_RW(devaddr, rsvd[6], 0, 0x7f)
282
283static inline u64 ep_ctx_read_deq_ptr(struct tegra_xudc_ep_context *ctx)
284{
285 return ((u64)ep_ctx_read_deq_hi(ctx) << 32) |
286 (ep_ctx_read_deq_lo(ctx) << 4);
287}
288
289static inline void
290ep_ctx_write_deq_ptr(struct tegra_xudc_ep_context *ctx, u64 addr)
291{
292 ep_ctx_write_deq_lo(ctx, lower_32_bits(addr) >> 4);
293 ep_ctx_write_deq_hi(ctx, upper_32_bits(addr));
294}
295
296struct tegra_xudc_trb {
297 __le32 data_lo;
298 __le32 data_hi;
299 __le32 status;
300 __le32 control;
301};
302
303#define TRB_TYPE_RSVD 0
304#define TRB_TYPE_NORMAL 1
305#define TRB_TYPE_SETUP_STAGE 2
306#define TRB_TYPE_DATA_STAGE 3
307#define TRB_TYPE_STATUS_STAGE 4
308#define TRB_TYPE_ISOCH 5
309#define TRB_TYPE_LINK 6
310#define TRB_TYPE_TRANSFER_EVENT 32
311#define TRB_TYPE_PORT_STATUS_CHANGE_EVENT 34
312#define TRB_TYPE_STREAM 48
313#define TRB_TYPE_SETUP_PACKET_EVENT 63
314
315#define TRB_CMPL_CODE_INVALID 0
316#define TRB_CMPL_CODE_SUCCESS 1
317#define TRB_CMPL_CODE_DATA_BUFFER_ERR 2
318#define TRB_CMPL_CODE_BABBLE_DETECTED_ERR 3
319#define TRB_CMPL_CODE_USB_TRANS_ERR 4
320#define TRB_CMPL_CODE_TRB_ERR 5
321#define TRB_CMPL_CODE_STALL 6
322#define TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR 10
323#define TRB_CMPL_CODE_SHORT_PACKET 13
324#define TRB_CMPL_CODE_RING_UNDERRUN 14
325#define TRB_CMPL_CODE_RING_OVERRUN 15
326#define TRB_CMPL_CODE_EVENT_RING_FULL_ERR 21
327#define TRB_CMPL_CODE_STOPPED 26
328#define TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN 31
329#define TRB_CMPL_CODE_STREAM_NUMP_ERROR 219
330#define TRB_CMPL_CODE_PRIME_PIPE_RECEIVED 220
331#define TRB_CMPL_CODE_HOST_REJECTED 221
332#define TRB_CMPL_CODE_CTRL_DIR_ERR 222
333#define TRB_CMPL_CODE_CTRL_SEQNUM_ERR 223
334
335#define BUILD_TRB_RW(name, member, shift, mask) \
336static inline u32 trb_read_##name(struct tegra_xudc_trb *trb) \
337{ \
338 return (le32_to_cpu(trb->member) >> (shift)) & (mask); \
339} \
340static inline void \
341trb_write_##name(struct tegra_xudc_trb *trb, u32 val) \
342{ \
343 u32 tmp; \
344 \
345 tmp = le32_to_cpu(trb->member) & ~((mask) << (shift)); \
346 tmp |= (val & (mask)) << (shift); \
347 trb->member = cpu_to_le32(tmp); \
348}
349
350BUILD_TRB_RW(data_lo, data_lo, 0, 0xffffffff)
351BUILD_TRB_RW(data_hi, data_hi, 0, 0xffffffff)
352BUILD_TRB_RW(seq_num, status, 0, 0xffff)
353BUILD_TRB_RW(transfer_len, status, 0, 0xffffff)
354BUILD_TRB_RW(td_size, status, 17, 0x1f)
355BUILD_TRB_RW(cmpl_code, status, 24, 0xff)
356BUILD_TRB_RW(cycle, control, 0, 0x1)
357BUILD_TRB_RW(toggle_cycle, control, 1, 0x1)
358BUILD_TRB_RW(isp, control, 2, 0x1)
359BUILD_TRB_RW(chain, control, 4, 0x1)
360BUILD_TRB_RW(ioc, control, 5, 0x1)
361BUILD_TRB_RW(type, control, 10, 0x3f)
362BUILD_TRB_RW(stream_id, control, 16, 0xffff)
363BUILD_TRB_RW(endpoint_id, control, 16, 0x1f)
364BUILD_TRB_RW(tlbpc, control, 16, 0xf)
365BUILD_TRB_RW(data_stage_dir, control, 16, 0x1)
366BUILD_TRB_RW(frame_id, control, 20, 0x7ff)
367BUILD_TRB_RW(sia, control, 31, 0x1)
368
369static inline u64 trb_read_data_ptr(struct tegra_xudc_trb *trb)
370{
371 return ((u64)trb_read_data_hi(trb) << 32) |
372 trb_read_data_lo(trb);
373}
374
375static inline void trb_write_data_ptr(struct tegra_xudc_trb *trb, u64 addr)
376{
377 trb_write_data_lo(trb, lower_32_bits(addr));
378 trb_write_data_hi(trb, upper_32_bits(addr));
379}
380
381struct tegra_xudc_request {
382 struct usb_request usb_req;
383
384 size_t buf_queued;
385 unsigned int trbs_queued;
386 unsigned int trbs_needed;
387 bool need_zlp;
388
389 struct tegra_xudc_trb *first_trb;
390 struct tegra_xudc_trb *last_trb;
391
392 struct list_head list;
393};
394
395struct tegra_xudc_ep {
396 struct tegra_xudc *xudc;
397 struct usb_ep usb_ep;
398 unsigned int index;
399 char name[8];
400
401 struct tegra_xudc_ep_context *context;
402
403#define XUDC_TRANSFER_RING_SIZE 64
404 struct tegra_xudc_trb *transfer_ring;
405 dma_addr_t transfer_ring_phys;
406
407 unsigned int enq_ptr;
408 unsigned int deq_ptr;
409 bool pcs;
410 bool ring_full;
411 bool stream_rejected;
412
413 struct list_head queue;
414 const struct usb_endpoint_descriptor *desc;
415 const struct usb_ss_ep_comp_descriptor *comp_desc;
416};
417
418struct tegra_xudc_sel_timing {
419 __u8 u1sel;
420 __u8 u1pel;
421 __le16 u2sel;
422 __le16 u2pel;
423};
424
425enum tegra_xudc_setup_state {
426 WAIT_FOR_SETUP,
427 DATA_STAGE_XFER,
428 DATA_STAGE_RECV,
429 STATUS_STAGE_XFER,
430 STATUS_STAGE_RECV,
431};
432
433struct tegra_xudc_setup_packet {
434 struct usb_ctrlrequest ctrl_req;
435 unsigned int seq_num;
436};
437
438struct tegra_xudc_save_regs {
439 u32 ctrl;
440 u32 portpm;
441};
442
443struct tegra_xudc {
444 struct device *dev;
445 const struct tegra_xudc_soc *soc;
446 struct tegra_xusb_padctl *padctl;
447
448 spinlock_t lock;
449
450 struct usb_gadget gadget;
451 struct usb_gadget_driver *driver;
452
453#define XUDC_NR_EVENT_RINGS 2
454#define XUDC_EVENT_RING_SIZE 4096
455 struct tegra_xudc_trb *event_ring[XUDC_NR_EVENT_RINGS];
456 dma_addr_t event_ring_phys[XUDC_NR_EVENT_RINGS];
457 unsigned int event_ring_index;
458 unsigned int event_ring_deq_ptr;
459 bool ccs;
460
461#define XUDC_NR_EPS 32
462 struct tegra_xudc_ep ep[XUDC_NR_EPS];
463 struct tegra_xudc_ep_context *ep_context;
464 dma_addr_t ep_context_phys;
465
466 struct device *genpd_dev_device;
467 struct device *genpd_dev_ss;
468 struct device_link *genpd_dl_device;
469 struct device_link *genpd_dl_ss;
470
471 struct dma_pool *transfer_ring_pool;
472
473 bool queued_setup_packet;
474 struct tegra_xudc_setup_packet setup_packet;
475 enum tegra_xudc_setup_state setup_state;
476 u16 setup_seq_num;
477
478 u16 dev_addr;
479 u16 isoch_delay;
480 struct tegra_xudc_sel_timing sel_timing;
481 u8 test_mode_pattern;
482 u16 status_buf;
483 struct tegra_xudc_request *ep0_req;
484
485 bool pullup;
486
487 unsigned int nr_enabled_eps;
488 unsigned int nr_isoch_eps;
489
490 unsigned int device_state;
491 unsigned int resume_state;
492
493 int irq;
494
495 void __iomem *base;
496 resource_size_t phys_base;
497 void __iomem *ipfs;
498 void __iomem *fpci;
499
500 struct regulator_bulk_data *supplies;
501
502 struct clk_bulk_data *clks;
503
504 bool device_mode;
505 struct work_struct usb_role_sw_work;
506
507 struct phy **usb3_phy;
508 struct phy *curr_usb3_phy;
509 struct phy **utmi_phy;
510 struct phy *curr_utmi_phy;
511
512 struct tegra_xudc_save_regs saved_regs;
513 bool suspended;
514 bool powergated;
515
516 struct usb_phy **usbphy;
517 struct usb_phy *curr_usbphy;
518 struct notifier_block vbus_nb;
519
520 struct completion disconnect_complete;
521
522 bool selfpowered;
523
524#define TOGGLE_VBUS_WAIT_MS 100
525 struct delayed_work plc_reset_work;
526 bool wait_csc;
527
528 struct delayed_work port_reset_war_work;
529 bool wait_for_sec_prc;
530};
531
532#define XUDC_TRB_MAX_BUFFER_SIZE 65536
533#define XUDC_MAX_ISOCH_EPS 4
534#define XUDC_INTERRUPT_MODERATION_US 0
535
536static struct usb_endpoint_descriptor tegra_xudc_ep0_desc = {
537 .bLength = USB_DT_ENDPOINT_SIZE,
538 .bDescriptorType = USB_DT_ENDPOINT,
539 .bEndpointAddress = 0,
540 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
541 .wMaxPacketSize = cpu_to_le16(64),
542};
543
544struct tegra_xudc_soc {
545 const char * const *supply_names;
546 unsigned int num_supplies;
547 const char * const *clock_names;
548 unsigned int num_clks;
549 unsigned int num_phys;
550 bool u1_enable;
551 bool u2_enable;
552 bool lpm_enable;
553 bool invalid_seq_num;
554 bool pls_quirk;
555 bool port_reset_quirk;
556 bool port_speed_quirk;
557 bool has_ipfs;
558};
559
560static inline u32 fpci_readl(struct tegra_xudc *xudc, unsigned int offset)
561{
562 return readl(xudc->fpci + offset);
563}
564
565static inline void fpci_writel(struct tegra_xudc *xudc, u32 val,
566 unsigned int offset)
567{
568 writel(val, xudc->fpci + offset);
569}
570
571static inline u32 ipfs_readl(struct tegra_xudc *xudc, unsigned int offset)
572{
573 return readl(xudc->ipfs + offset);
574}
575
576static inline void ipfs_writel(struct tegra_xudc *xudc, u32 val,
577 unsigned int offset)
578{
579 writel(val, xudc->ipfs + offset);
580}
581
582static inline u32 xudc_readl(struct tegra_xudc *xudc, unsigned int offset)
583{
584 return readl(xudc->base + offset);
585}
586
587static inline void xudc_writel(struct tegra_xudc *xudc, u32 val,
588 unsigned int offset)
589{
590 writel(val, xudc->base + offset);
591}
592
593static inline int xudc_readl_poll(struct tegra_xudc *xudc,
594 unsigned int offset, u32 mask, u32 val)
595{
596 u32 regval;
597
598 return readl_poll_timeout_atomic(xudc->base + offset, regval,
599 (regval & mask) == val, 1, 100);
600}
601
602static inline struct tegra_xudc *to_xudc(struct usb_gadget *gadget)
603{
604 return container_of(gadget, struct tegra_xudc, gadget);
605}
606
607static inline struct tegra_xudc_ep *to_xudc_ep(struct usb_ep *ep)
608{
609 return container_of(ep, struct tegra_xudc_ep, usb_ep);
610}
611
612static inline struct tegra_xudc_request *to_xudc_req(struct usb_request *req)
613{
614 return container_of(req, struct tegra_xudc_request, usb_req);
615}
616
617static inline void dump_trb(struct tegra_xudc *xudc, const char *type,
618 struct tegra_xudc_trb *trb)
619{
620 dev_dbg(xudc->dev,
621 "%s: %p, lo = %#x, hi = %#x, status = %#x, control = %#x\n",
622 type, trb, trb->data_lo, trb->data_hi, trb->status,
623 trb->control);
624}
625
626static void tegra_xudc_limit_port_speed(struct tegra_xudc *xudc)
627{
628 u32 val;
629
630 /* limit port speed to gen 1 */
631 val = xudc_readl(xudc, SSPX_CORE_CNT56);
632 val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
633 val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x260);
634 xudc_writel(xudc, val, SSPX_CORE_CNT56);
635
636 val = xudc_readl(xudc, SSPX_CORE_CNT57);
637 val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
638 val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x6D6);
639 xudc_writel(xudc, val, SSPX_CORE_CNT57);
640
641 val = xudc_readl(xudc, SSPX_CORE_CNT65);
642 val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
643 val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0x4B0);
644 xudc_writel(xudc, val, SSPX_CORE_CNT66);
645
646 val = xudc_readl(xudc, SSPX_CORE_CNT66);
647 val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
648 val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x4B0);
649 xudc_writel(xudc, val, SSPX_CORE_CNT66);
650
651 val = xudc_readl(xudc, SSPX_CORE_CNT67);
652 val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
653 val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x4B0);
654 xudc_writel(xudc, val, SSPX_CORE_CNT67);
655
656 val = xudc_readl(xudc, SSPX_CORE_CNT72);
657 val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
658 val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x10);
659 xudc_writel(xudc, val, SSPX_CORE_CNT72);
660}
661
662static void tegra_xudc_restore_port_speed(struct tegra_xudc *xudc)
663{
664 u32 val;
665
666 /* restore port speed to gen2 */
667 val = xudc_readl(xudc, SSPX_CORE_CNT56);
668 val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
669 val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x438);
670 xudc_writel(xudc, val, SSPX_CORE_CNT56);
671
672 val = xudc_readl(xudc, SSPX_CORE_CNT57);
673 val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
674 val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x528);
675 xudc_writel(xudc, val, SSPX_CORE_CNT57);
676
677 val = xudc_readl(xudc, SSPX_CORE_CNT65);
678 val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
679 val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0xE10);
680 xudc_writel(xudc, val, SSPX_CORE_CNT66);
681
682 val = xudc_readl(xudc, SSPX_CORE_CNT66);
683 val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
684 val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x348);
685 xudc_writel(xudc, val, SSPX_CORE_CNT66);
686
687 val = xudc_readl(xudc, SSPX_CORE_CNT67);
688 val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
689 val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x5a0);
690 xudc_writel(xudc, val, SSPX_CORE_CNT67);
691
692 val = xudc_readl(xudc, SSPX_CORE_CNT72);
693 val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
694 val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x1c21);
695 xudc_writel(xudc, val, SSPX_CORE_CNT72);
696}
697
698static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc)
699{
700 int err;
701
702 pm_runtime_get_sync(xudc->dev);
703
704 tegra_phy_xusb_utmi_pad_power_on(xudc->curr_utmi_phy);
705
706 err = phy_power_on(xudc->curr_utmi_phy);
707 if (err < 0)
708 dev_err(xudc->dev, "UTMI power on failed: %d\n", err);
709
710 err = phy_power_on(xudc->curr_usb3_phy);
711 if (err < 0)
712 dev_err(xudc->dev, "USB3 PHY power on failed: %d\n", err);
713
714 dev_dbg(xudc->dev, "device mode on\n");
715
716 phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
717 USB_ROLE_DEVICE);
718}
719
720static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc)
721{
722 bool connected = false;
723 u32 pls, val;
724 int err;
725
726 dev_dbg(xudc->dev, "device mode off\n");
727
728 connected = !!(xudc_readl(xudc, PORTSC) & PORTSC_CCS);
729
730 reinit_completion(&xudc->disconnect_complete);
731
732 if (xudc->soc->port_speed_quirk)
733 tegra_xudc_restore_port_speed(xudc);
734
735 phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG, USB_ROLE_NONE);
736
737 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
738 PORTSC_PLS_SHIFT;
739
740 /* Direct link to U0 if disconnected in RESUME or U2. */
741 if (xudc->soc->pls_quirk && xudc->gadget.speed == USB_SPEED_SUPER &&
742 (pls == PORTSC_PLS_RESUME || pls == PORTSC_PLS_U2)) {
743 val = xudc_readl(xudc, PORTPM);
744 val |= PORTPM_FRWE;
745 xudc_writel(xudc, val, PORTPM);
746
747 val = xudc_readl(xudc, PORTSC);
748 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
749 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
750 xudc_writel(xudc, val, PORTSC);
751 }
752
753 /* Wait for disconnect event. */
754 if (connected)
755 wait_for_completion(&xudc->disconnect_complete);
756
757 /* Make sure interrupt handler has completed before powergating. */
758 synchronize_irq(xudc->irq);
759
760 tegra_phy_xusb_utmi_pad_power_down(xudc->curr_utmi_phy);
761
762 err = phy_power_off(xudc->curr_utmi_phy);
763 if (err < 0)
764 dev_err(xudc->dev, "UTMI PHY power off failed: %d\n", err);
765
766 err = phy_power_off(xudc->curr_usb3_phy);
767 if (err < 0)
768 dev_err(xudc->dev, "USB3 PHY power off failed: %d\n", err);
769
770 pm_runtime_put(xudc->dev);
771}
772
773static void tegra_xudc_usb_role_sw_work(struct work_struct *work)
774{
775 struct tegra_xudc *xudc = container_of(work, struct tegra_xudc,
776 usb_role_sw_work);
777
778 if (xudc->device_mode)
779 tegra_xudc_device_mode_on(xudc);
780 else
781 tegra_xudc_device_mode_off(xudc);
782}
783
784static int tegra_xudc_get_phy_index(struct tegra_xudc *xudc,
785 struct usb_phy *usbphy)
786{
787 unsigned int i;
788
789 for (i = 0; i < xudc->soc->num_phys; i++) {
790 if (xudc->usbphy[i] && usbphy == xudc->usbphy[i])
791 return i;
792 }
793
794 dev_info(xudc->dev, "phy index could not be found for shared USB PHY");
795 return -1;
796}
797
798static void tegra_xudc_update_data_role(struct tegra_xudc *xudc,
799 struct usb_phy *usbphy)
800{
801 int phy_index;
802
803 if ((xudc->device_mode && usbphy->last_event == USB_EVENT_VBUS) ||
804 (!xudc->device_mode && usbphy->last_event != USB_EVENT_VBUS)) {
805 dev_dbg(xudc->dev, "Same role(%d) received. Ignore",
806 xudc->device_mode);
807 return;
808 }
809
810 xudc->device_mode = (usbphy->last_event == USB_EVENT_VBUS) ? true :
811 false;
812
813 phy_index = tegra_xudc_get_phy_index(xudc, usbphy);
814 dev_dbg(xudc->dev, "%s(): current phy index is %d\n", __func__,
815 phy_index);
816
817 if (!xudc->suspended && phy_index != -1) {
818 xudc->curr_utmi_phy = xudc->utmi_phy[phy_index];
819 xudc->curr_usb3_phy = xudc->usb3_phy[phy_index];
820 xudc->curr_usbphy = usbphy;
821 schedule_work(&xudc->usb_role_sw_work);
822 }
823}
824
825static int tegra_xudc_vbus_notify(struct notifier_block *nb,
826 unsigned long action, void *data)
827{
828 struct tegra_xudc *xudc = container_of(nb, struct tegra_xudc,
829 vbus_nb);
830 struct usb_phy *usbphy = (struct usb_phy *)data;
831
832 dev_dbg(xudc->dev, "%s(): event is %d\n", __func__, usbphy->last_event);
833
834 tegra_xudc_update_data_role(xudc, usbphy);
835
836 return NOTIFY_OK;
837}
838
839static void tegra_xudc_plc_reset_work(struct work_struct *work)
840{
841 struct delayed_work *dwork = to_delayed_work(work);
842 struct tegra_xudc *xudc = container_of(dwork, struct tegra_xudc,
843 plc_reset_work);
844 unsigned long flags;
845
846 spin_lock_irqsave(&xudc->lock, flags);
847
848 if (xudc->wait_csc) {
849 u32 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
850 PORTSC_PLS_SHIFT;
851
852 if (pls == PORTSC_PLS_INACTIVE) {
853 dev_info(xudc->dev, "PLS = Inactive. Toggle VBUS\n");
854 phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
855 USB_ROLE_NONE);
856 phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
857 USB_ROLE_DEVICE);
858
859 xudc->wait_csc = false;
860 }
861 }
862
863 spin_unlock_irqrestore(&xudc->lock, flags);
864}
865
866static void tegra_xudc_port_reset_war_work(struct work_struct *work)
867{
868 struct delayed_work *dwork = to_delayed_work(work);
869 struct tegra_xudc *xudc =
870 container_of(dwork, struct tegra_xudc, port_reset_war_work);
871 unsigned long flags;
872 u32 pls;
873 int ret;
874
875 spin_lock_irqsave(&xudc->lock, flags);
876
877 if (xudc->device_mode && xudc->wait_for_sec_prc) {
878 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
879 PORTSC_PLS_SHIFT;
880 dev_dbg(xudc->dev, "pls = %x\n", pls);
881
882 if (pls == PORTSC_PLS_DISABLED) {
883 dev_dbg(xudc->dev, "toggle vbus\n");
884 /* PRC doesn't complete in 100ms, toggle the vbus */
885 ret = tegra_phy_xusb_utmi_port_reset(
886 xudc->curr_utmi_phy);
887 if (ret == 1)
888 xudc->wait_for_sec_prc = 0;
889 }
890 }
891
892 spin_unlock_irqrestore(&xudc->lock, flags);
893}
894
895static dma_addr_t trb_virt_to_phys(struct tegra_xudc_ep *ep,
896 struct tegra_xudc_trb *trb)
897{
898 unsigned int index;
899
900 index = trb - ep->transfer_ring;
901
902 if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
903 return 0;
904
905 return (ep->transfer_ring_phys + index * sizeof(*trb));
906}
907
908static struct tegra_xudc_trb *trb_phys_to_virt(struct tegra_xudc_ep *ep,
909 dma_addr_t addr)
910{
911 struct tegra_xudc_trb *trb;
912 unsigned int index;
913
914 index = (addr - ep->transfer_ring_phys) / sizeof(*trb);
915
916 if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
917 return NULL;
918
919 trb = &ep->transfer_ring[index];
920
921 return trb;
922}
923
924static void ep_reload(struct tegra_xudc *xudc, unsigned int ep)
925{
926 xudc_writel(xudc, BIT(ep), EP_RELOAD);
927 xudc_readl_poll(xudc, EP_RELOAD, BIT(ep), 0);
928}
929
930static void ep_pause(struct tegra_xudc *xudc, unsigned int ep)
931{
932 u32 val;
933
934 val = xudc_readl(xudc, EP_PAUSE);
935 if (val & BIT(ep))
936 return;
937 val |= BIT(ep);
938
939 xudc_writel(xudc, val, EP_PAUSE);
940
941 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
942
943 xudc_writel(xudc, BIT(ep), EP_STCHG);
944}
945
946static void ep_unpause(struct tegra_xudc *xudc, unsigned int ep)
947{
948 u32 val;
949
950 val = xudc_readl(xudc, EP_PAUSE);
951 if (!(val & BIT(ep)))
952 return;
953 val &= ~BIT(ep);
954
955 xudc_writel(xudc, val, EP_PAUSE);
956
957 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
958
959 xudc_writel(xudc, BIT(ep), EP_STCHG);
960}
961
962static void ep_unpause_all(struct tegra_xudc *xudc)
963{
964 u32 val;
965
966 val = xudc_readl(xudc, EP_PAUSE);
967
968 xudc_writel(xudc, 0, EP_PAUSE);
969
970 xudc_readl_poll(xudc, EP_STCHG, val, val);
971
972 xudc_writel(xudc, val, EP_STCHG);
973}
974
975static void ep_halt(struct tegra_xudc *xudc, unsigned int ep)
976{
977 u32 val;
978
979 val = xudc_readl(xudc, EP_HALT);
980 if (val & BIT(ep))
981 return;
982 val |= BIT(ep);
983 xudc_writel(xudc, val, EP_HALT);
984
985 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
986
987 xudc_writel(xudc, BIT(ep), EP_STCHG);
988}
989
990static void ep_unhalt(struct tegra_xudc *xudc, unsigned int ep)
991{
992 u32 val;
993
994 val = xudc_readl(xudc, EP_HALT);
995 if (!(val & BIT(ep)))
996 return;
997 val &= ~BIT(ep);
998 xudc_writel(xudc, val, EP_HALT);
999
1000 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
1001
1002 xudc_writel(xudc, BIT(ep), EP_STCHG);
1003}
1004
1005static void ep_unhalt_all(struct tegra_xudc *xudc)
1006{
1007 u32 val;
1008
1009 val = xudc_readl(xudc, EP_HALT);
1010 if (!val)
1011 return;
1012 xudc_writel(xudc, 0, EP_HALT);
1013
1014 xudc_readl_poll(xudc, EP_STCHG, val, val);
1015
1016 xudc_writel(xudc, val, EP_STCHG);
1017}
1018
1019static void ep_wait_for_stopped(struct tegra_xudc *xudc, unsigned int ep)
1020{
1021 xudc_readl_poll(xudc, EP_STOPPED, BIT(ep), BIT(ep));
1022 xudc_writel(xudc, BIT(ep), EP_STOPPED);
1023}
1024
1025static void ep_wait_for_inactive(struct tegra_xudc *xudc, unsigned int ep)
1026{
1027 xudc_readl_poll(xudc, EP_THREAD_ACTIVE, BIT(ep), 0);
1028}
1029
1030static void tegra_xudc_req_done(struct tegra_xudc_ep *ep,
1031 struct tegra_xudc_request *req, int status)
1032{
1033 struct tegra_xudc *xudc = ep->xudc;
1034
1035 dev_dbg(xudc->dev, "completing request %p on EP %u with status %d\n",
1036 req, ep->index, status);
1037
1038 if (likely(req->usb_req.status == -EINPROGRESS))
1039 req->usb_req.status = status;
1040
1041 list_del_init(&req->list);
1042
1043 if (usb_endpoint_xfer_control(ep->desc)) {
1044 usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
1045 (xudc->setup_state ==
1046 DATA_STAGE_XFER));
1047 } else {
1048 usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
1049 usb_endpoint_dir_in(ep->desc));
1050 }
1051
1052 spin_unlock(&xudc->lock);
1053 usb_gadget_giveback_request(&ep->usb_ep, &req->usb_req);
1054 spin_lock(&xudc->lock);
1055}
1056
1057static void tegra_xudc_ep_nuke(struct tegra_xudc_ep *ep, int status)
1058{
1059 struct tegra_xudc_request *req;
1060
1061 while (!list_empty(&ep->queue)) {
1062 req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1063 list);
1064 tegra_xudc_req_done(ep, req, status);
1065 }
1066}
1067
1068static unsigned int ep_available_trbs(struct tegra_xudc_ep *ep)
1069{
1070 if (ep->ring_full)
1071 return 0;
1072
1073 if (ep->deq_ptr > ep->enq_ptr)
1074 return ep->deq_ptr - ep->enq_ptr - 1;
1075
1076 return XUDC_TRANSFER_RING_SIZE - (ep->enq_ptr - ep->deq_ptr) - 2;
1077}
1078
1079static void tegra_xudc_queue_one_trb(struct tegra_xudc_ep *ep,
1080 struct tegra_xudc_request *req,
1081 struct tegra_xudc_trb *trb,
1082 bool ioc)
1083{
1084 struct tegra_xudc *xudc = ep->xudc;
1085 dma_addr_t buf_addr;
1086 size_t len;
1087
1088 len = min_t(size_t, XUDC_TRB_MAX_BUFFER_SIZE, req->usb_req.length -
1089 req->buf_queued);
1090 if (len > 0)
1091 buf_addr = req->usb_req.dma + req->buf_queued;
1092 else
1093 buf_addr = 0;
1094
1095 trb_write_data_ptr(trb, buf_addr);
1096
1097 trb_write_transfer_len(trb, len);
1098 trb_write_td_size(trb, req->trbs_needed - req->trbs_queued - 1);
1099
1100 if (req->trbs_queued == req->trbs_needed - 1 ||
1101 (req->need_zlp && req->trbs_queued == req->trbs_needed - 2))
1102 trb_write_chain(trb, 0);
1103 else
1104 trb_write_chain(trb, 1);
1105
1106 trb_write_ioc(trb, ioc);
1107
1108 if (usb_endpoint_dir_out(ep->desc) ||
1109 (usb_endpoint_xfer_control(ep->desc) &&
1110 (xudc->setup_state == DATA_STAGE_RECV)))
1111 trb_write_isp(trb, 1);
1112 else
1113 trb_write_isp(trb, 0);
1114
1115 if (usb_endpoint_xfer_control(ep->desc)) {
1116 if (xudc->setup_state == DATA_STAGE_XFER ||
1117 xudc->setup_state == DATA_STAGE_RECV)
1118 trb_write_type(trb, TRB_TYPE_DATA_STAGE);
1119 else
1120 trb_write_type(trb, TRB_TYPE_STATUS_STAGE);
1121
1122 if (xudc->setup_state == DATA_STAGE_XFER ||
1123 xudc->setup_state == STATUS_STAGE_XFER)
1124 trb_write_data_stage_dir(trb, 1);
1125 else
1126 trb_write_data_stage_dir(trb, 0);
1127 } else if (usb_endpoint_xfer_isoc(ep->desc)) {
1128 trb_write_type(trb, TRB_TYPE_ISOCH);
1129 trb_write_sia(trb, 1);
1130 trb_write_frame_id(trb, 0);
1131 trb_write_tlbpc(trb, 0);
1132 } else if (usb_ss_max_streams(ep->comp_desc)) {
1133 trb_write_type(trb, TRB_TYPE_STREAM);
1134 trb_write_stream_id(trb, req->usb_req.stream_id);
1135 } else {
1136 trb_write_type(trb, TRB_TYPE_NORMAL);
1137 trb_write_stream_id(trb, 0);
1138 }
1139
1140 trb_write_cycle(trb, ep->pcs);
1141
1142 req->trbs_queued++;
1143 req->buf_queued += len;
1144
1145 dump_trb(xudc, "TRANSFER", trb);
1146}
1147
1148static unsigned int tegra_xudc_queue_trbs(struct tegra_xudc_ep *ep,
1149 struct tegra_xudc_request *req)
1150{
1151 unsigned int i, count, available;
1152 bool wait_td = false;
1153
1154 available = ep_available_trbs(ep);
1155 count = req->trbs_needed - req->trbs_queued;
1156 if (available < count) {
1157 count = available;
1158 ep->ring_full = true;
1159 }
1160
1161 /*
1162 * To generate zero-length packet on USB bus, SW needs schedule a
1163 * standalone zero-length TD. According to HW's behavior, SW needs
1164 * to schedule TDs in different ways for different endpoint types.
1165 *
1166 * For control endpoint:
1167 * - Data stage TD (IOC = 1, CH = 0)
1168 * - Ring doorbell and wait transfer event
1169 * - Data stage TD for ZLP (IOC = 1, CH = 0)
1170 * - Ring doorbell
1171 *
1172 * For bulk and interrupt endpoints:
1173 * - Normal transfer TD (IOC = 0, CH = 0)
1174 * - Normal transfer TD for ZLP (IOC = 1, CH = 0)
1175 * - Ring doorbell
1176 */
1177
1178 if (req->need_zlp && usb_endpoint_xfer_control(ep->desc) && count > 1)
1179 wait_td = true;
1180
1181 if (!req->first_trb)
1182 req->first_trb = &ep->transfer_ring[ep->enq_ptr];
1183
1184 for (i = 0; i < count; i++) {
1185 struct tegra_xudc_trb *trb = &ep->transfer_ring[ep->enq_ptr];
1186 bool ioc = false;
1187
1188 if ((i == count - 1) || (wait_td && i == count - 2))
1189 ioc = true;
1190
1191 tegra_xudc_queue_one_trb(ep, req, trb, ioc);
1192 req->last_trb = trb;
1193
1194 ep->enq_ptr++;
1195 if (ep->enq_ptr == XUDC_TRANSFER_RING_SIZE - 1) {
1196 trb = &ep->transfer_ring[ep->enq_ptr];
1197 trb_write_cycle(trb, ep->pcs);
1198 ep->pcs = !ep->pcs;
1199 ep->enq_ptr = 0;
1200 }
1201
1202 if (ioc)
1203 break;
1204 }
1205
1206 return count;
1207}
1208
1209static void tegra_xudc_ep_ring_doorbell(struct tegra_xudc_ep *ep)
1210{
1211 struct tegra_xudc *xudc = ep->xudc;
1212 u32 val;
1213
1214 if (list_empty(&ep->queue))
1215 return;
1216
1217 val = DB_TARGET(ep->index);
1218 if (usb_endpoint_xfer_control(ep->desc)) {
1219 val |= DB_STREAMID(xudc->setup_seq_num);
1220 } else if (usb_ss_max_streams(ep->comp_desc) > 0) {
1221 struct tegra_xudc_request *req;
1222
1223 /* Don't ring doorbell if the stream has been rejected. */
1224 if (ep->stream_rejected)
1225 return;
1226
1227 req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1228 list);
1229 val |= DB_STREAMID(req->usb_req.stream_id);
1230 }
1231
1232 dev_dbg(xudc->dev, "ring doorbell: %#x\n", val);
1233 xudc_writel(xudc, val, DB);
1234}
1235
1236static void tegra_xudc_ep_kick_queue(struct tegra_xudc_ep *ep)
1237{
1238 struct tegra_xudc_request *req;
1239 bool trbs_queued = false;
1240
1241 list_for_each_entry(req, &ep->queue, list) {
1242 if (ep->ring_full)
1243 break;
1244
1245 if (tegra_xudc_queue_trbs(ep, req) > 0)
1246 trbs_queued = true;
1247 }
1248
1249 if (trbs_queued)
1250 tegra_xudc_ep_ring_doorbell(ep);
1251}
1252
1253static int
1254__tegra_xudc_ep_queue(struct tegra_xudc_ep *ep, struct tegra_xudc_request *req)
1255{
1256 struct tegra_xudc *xudc = ep->xudc;
1257 int err;
1258
1259 if (usb_endpoint_xfer_control(ep->desc) && !list_empty(&ep->queue)) {
1260 dev_err(xudc->dev, "control EP has pending transfers\n");
1261 return -EINVAL;
1262 }
1263
1264 if (usb_endpoint_xfer_control(ep->desc)) {
1265 err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1266 (xudc->setup_state ==
1267 DATA_STAGE_XFER));
1268 } else {
1269 err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1270 usb_endpoint_dir_in(ep->desc));
1271 }
1272
1273 if (err < 0) {
1274 dev_err(xudc->dev, "failed to map request: %d\n", err);
1275 return err;
1276 }
1277
1278 req->first_trb = NULL;
1279 req->last_trb = NULL;
1280 req->buf_queued = 0;
1281 req->trbs_queued = 0;
1282 req->need_zlp = false;
1283 req->trbs_needed = DIV_ROUND_UP(req->usb_req.length,
1284 XUDC_TRB_MAX_BUFFER_SIZE);
1285 if (req->usb_req.length == 0)
1286 req->trbs_needed++;
1287
1288 if (!usb_endpoint_xfer_isoc(ep->desc) &&
1289 req->usb_req.zero && req->usb_req.length &&
1290 ((req->usb_req.length % ep->usb_ep.maxpacket) == 0)) {
1291 req->trbs_needed++;
1292 req->need_zlp = true;
1293 }
1294
1295 req->usb_req.status = -EINPROGRESS;
1296 req->usb_req.actual = 0;
1297
1298 list_add_tail(&req->list, &ep->queue);
1299
1300 tegra_xudc_ep_kick_queue(ep);
1301
1302 return 0;
1303}
1304
1305static int
1306tegra_xudc_ep_queue(struct usb_ep *usb_ep, struct usb_request *usb_req,
1307 gfp_t gfp)
1308{
1309 struct tegra_xudc_request *req;
1310 struct tegra_xudc_ep *ep;
1311 struct tegra_xudc *xudc;
1312 unsigned long flags;
1313 int ret;
1314
1315 if (!usb_ep || !usb_req)
1316 return -EINVAL;
1317
1318 ep = to_xudc_ep(usb_ep);
1319 req = to_xudc_req(usb_req);
1320 xudc = ep->xudc;
1321
1322 spin_lock_irqsave(&xudc->lock, flags);
1323 if (xudc->powergated || !ep->desc) {
1324 ret = -ESHUTDOWN;
1325 goto unlock;
1326 }
1327
1328 ret = __tegra_xudc_ep_queue(ep, req);
1329unlock:
1330 spin_unlock_irqrestore(&xudc->lock, flags);
1331
1332 return ret;
1333}
1334
1335static void squeeze_transfer_ring(struct tegra_xudc_ep *ep,
1336 struct tegra_xudc_request *req)
1337{
1338 struct tegra_xudc_trb *trb = req->first_trb;
1339 bool pcs_enq = trb_read_cycle(trb);
1340 bool pcs;
1341
1342 /*
1343 * Clear out all the TRBs part of or after the cancelled request,
1344 * and must correct trb cycle bit to the last un-enqueued state.
1345 */
1346 while (trb != &ep->transfer_ring[ep->enq_ptr]) {
1347 pcs = trb_read_cycle(trb);
1348 memset(trb, 0, sizeof(*trb));
1349 trb_write_cycle(trb, !pcs);
1350 trb++;
1351
1352 if (trb_read_type(trb) == TRB_TYPE_LINK)
1353 trb = ep->transfer_ring;
1354 }
1355
1356 /* Requests will be re-queued at the start of the cancelled request. */
1357 ep->enq_ptr = req->first_trb - ep->transfer_ring;
1358 /*
1359 * Retrieve the correct cycle bit state from the first trb of
1360 * the cancelled request.
1361 */
1362 ep->pcs = pcs_enq;
1363 ep->ring_full = false;
1364 list_for_each_entry_continue(req, &ep->queue, list) {
1365 req->usb_req.status = -EINPROGRESS;
1366 req->usb_req.actual = 0;
1367
1368 req->first_trb = NULL;
1369 req->last_trb = NULL;
1370 req->buf_queued = 0;
1371 req->trbs_queued = 0;
1372 }
1373}
1374
1375/*
1376 * Determine if the given TRB is in the range [first trb, last trb] for the
1377 * given request.
1378 */
1379static bool trb_in_request(struct tegra_xudc_ep *ep,
1380 struct tegra_xudc_request *req,
1381 struct tegra_xudc_trb *trb)
1382{
1383 dev_dbg(ep->xudc->dev, "%s: request %p -> %p; trb %p\n", __func__,
1384 req->first_trb, req->last_trb, trb);
1385
1386 if (trb >= req->first_trb && (trb <= req->last_trb ||
1387 req->last_trb < req->first_trb))
1388 return true;
1389
1390 if (trb < req->first_trb && trb <= req->last_trb &&
1391 req->last_trb < req->first_trb)
1392 return true;
1393
1394 return false;
1395}
1396
1397/*
1398 * Determine if the given TRB is in the range [EP enqueue pointer, first TRB)
1399 * for the given endpoint and request.
1400 */
1401static bool trb_before_request(struct tegra_xudc_ep *ep,
1402 struct tegra_xudc_request *req,
1403 struct tegra_xudc_trb *trb)
1404{
1405 struct tegra_xudc_trb *enq_trb = &ep->transfer_ring[ep->enq_ptr];
1406
1407 dev_dbg(ep->xudc->dev, "%s: request %p -> %p; enq ptr: %p; trb %p\n",
1408 __func__, req->first_trb, req->last_trb, enq_trb, trb);
1409
1410 if (trb < req->first_trb && (enq_trb <= trb ||
1411 req->first_trb < enq_trb))
1412 return true;
1413
1414 if (trb > req->first_trb && req->first_trb < enq_trb && enq_trb <= trb)
1415 return true;
1416
1417 return false;
1418}
1419
1420static int
1421__tegra_xudc_ep_dequeue(struct tegra_xudc_ep *ep,
1422 struct tegra_xudc_request *req)
1423{
1424 struct tegra_xudc *xudc = ep->xudc;
1425 struct tegra_xudc_request *r = NULL, *iter;
1426 struct tegra_xudc_trb *deq_trb;
1427 bool busy, kick_queue = false;
1428 int ret = 0;
1429
1430 /* Make sure the request is actually queued to this endpoint. */
1431 list_for_each_entry(iter, &ep->queue, list) {
1432 if (iter != req)
1433 continue;
1434 r = iter;
1435 break;
1436 }
1437
1438 if (!r)
1439 return -EINVAL;
1440
1441 /* Request hasn't been queued in the transfer ring yet. */
1442 if (!req->trbs_queued) {
1443 tegra_xudc_req_done(ep, req, -ECONNRESET);
1444 return 0;
1445 }
1446
1447 /* Halt DMA for this endpoint. */
1448 if (ep_ctx_read_state(ep->context) == EP_STATE_RUNNING) {
1449 ep_pause(xudc, ep->index);
1450 ep_wait_for_inactive(xudc, ep->index);
1451 }
1452
1453 deq_trb = trb_phys_to_virt(ep, ep_ctx_read_deq_ptr(ep->context));
1454 /* Is the hardware processing the TRB at the dequeue pointer? */
1455 busy = (trb_read_cycle(deq_trb) == ep_ctx_read_dcs(ep->context));
1456
1457 if (trb_in_request(ep, req, deq_trb) && busy) {
1458 /*
1459 * Request has been partially completed or it hasn't
1460 * started processing yet.
1461 */
1462 dma_addr_t deq_ptr;
1463
1464 squeeze_transfer_ring(ep, req);
1465
1466 req->usb_req.actual = ep_ctx_read_edtla(ep->context);
1467 tegra_xudc_req_done(ep, req, -ECONNRESET);
1468 kick_queue = true;
1469
1470 /* EDTLA is > 0: request has been partially completed */
1471 if (req->usb_req.actual > 0) {
1472 /*
1473 * Abort the pending transfer and update the dequeue
1474 * pointer
1475 */
1476 ep_ctx_write_edtla(ep->context, 0);
1477 ep_ctx_write_partial_td(ep->context, 0);
1478 ep_ctx_write_data_offset(ep->context, 0);
1479
1480 deq_ptr = trb_virt_to_phys(ep,
1481 &ep->transfer_ring[ep->enq_ptr]);
1482
1483 if (dma_mapping_error(xudc->dev, deq_ptr)) {
1484 ret = -EINVAL;
1485 } else {
1486 ep_ctx_write_deq_ptr(ep->context, deq_ptr);
1487 ep_ctx_write_dcs(ep->context, ep->pcs);
1488 ep_reload(xudc, ep->index);
1489 }
1490 }
1491 } else if (trb_before_request(ep, req, deq_trb) && busy) {
1492 /* Request hasn't started processing yet. */
1493 squeeze_transfer_ring(ep, req);
1494
1495 tegra_xudc_req_done(ep, req, -ECONNRESET);
1496 kick_queue = true;
1497 } else {
1498 /*
1499 * Request has completed, but we haven't processed the
1500 * completion event yet.
1501 */
1502 tegra_xudc_req_done(ep, req, -ECONNRESET);
1503 ret = -EINVAL;
1504 }
1505
1506 /* Resume the endpoint. */
1507 ep_unpause(xudc, ep->index);
1508
1509 if (kick_queue)
1510 tegra_xudc_ep_kick_queue(ep);
1511
1512 return ret;
1513}
1514
1515static int
1516tegra_xudc_ep_dequeue(struct usb_ep *usb_ep, struct usb_request *usb_req)
1517{
1518 struct tegra_xudc_request *req;
1519 struct tegra_xudc_ep *ep;
1520 struct tegra_xudc *xudc;
1521 unsigned long flags;
1522 int ret;
1523
1524 if (!usb_ep || !usb_req)
1525 return -EINVAL;
1526
1527 ep = to_xudc_ep(usb_ep);
1528 req = to_xudc_req(usb_req);
1529 xudc = ep->xudc;
1530
1531 spin_lock_irqsave(&xudc->lock, flags);
1532
1533 if (xudc->powergated || !ep->desc) {
1534 ret = -ESHUTDOWN;
1535 goto unlock;
1536 }
1537
1538 ret = __tegra_xudc_ep_dequeue(ep, req);
1539unlock:
1540 spin_unlock_irqrestore(&xudc->lock, flags);
1541
1542 return ret;
1543}
1544
1545static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt)
1546{
1547 struct tegra_xudc *xudc = ep->xudc;
1548
1549 if (!ep->desc)
1550 return -EINVAL;
1551
1552 if (usb_endpoint_xfer_isoc(ep->desc)) {
1553 dev_err(xudc->dev, "can't halt isochronous EP\n");
1554 return -ENOTSUPP;
1555 }
1556
1557 if (!!(xudc_readl(xudc, EP_HALT) & BIT(ep->index)) == halt) {
1558 dev_dbg(xudc->dev, "EP %u already %s\n", ep->index,
1559 halt ? "halted" : "not halted");
1560 return 0;
1561 }
1562
1563 if (halt) {
1564 ep_halt(xudc, ep->index);
1565 } else {
1566 ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1567
1568 ep_reload(xudc, ep->index);
1569
1570 ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1571 ep_ctx_write_rsvd(ep->context, 0);
1572 ep_ctx_write_partial_td(ep->context, 0);
1573 ep_ctx_write_splitxstate(ep->context, 0);
1574 ep_ctx_write_seq_num(ep->context, 0);
1575
1576 ep_reload(xudc, ep->index);
1577 ep_unpause(xudc, ep->index);
1578 ep_unhalt(xudc, ep->index);
1579
1580 tegra_xudc_ep_ring_doorbell(ep);
1581 }
1582
1583 return 0;
1584}
1585
1586static int tegra_xudc_ep_set_halt(struct usb_ep *usb_ep, int value)
1587{
1588 struct tegra_xudc_ep *ep;
1589 struct tegra_xudc *xudc;
1590 unsigned long flags;
1591 int ret;
1592
1593 if (!usb_ep)
1594 return -EINVAL;
1595
1596 ep = to_xudc_ep(usb_ep);
1597 xudc = ep->xudc;
1598
1599 spin_lock_irqsave(&xudc->lock, flags);
1600 if (xudc->powergated) {
1601 ret = -ESHUTDOWN;
1602 goto unlock;
1603 }
1604
1605 if (value && usb_endpoint_dir_in(ep->desc) &&
1606 !list_empty(&ep->queue)) {
1607 dev_err(xudc->dev, "can't halt EP with requests pending\n");
1608 ret = -EAGAIN;
1609 goto unlock;
1610 }
1611
1612 ret = __tegra_xudc_ep_set_halt(ep, value);
1613unlock:
1614 spin_unlock_irqrestore(&xudc->lock, flags);
1615
1616 return ret;
1617}
1618
1619static void tegra_xudc_ep_context_setup(struct tegra_xudc_ep *ep)
1620{
1621 const struct usb_endpoint_descriptor *desc = ep->desc;
1622 const struct usb_ss_ep_comp_descriptor *comp_desc = ep->comp_desc;
1623 struct tegra_xudc *xudc = ep->xudc;
1624 u16 maxpacket, maxburst = 0, esit = 0;
1625 u32 val;
1626
1627 maxpacket = usb_endpoint_maxp(desc);
1628 if (xudc->gadget.speed == USB_SPEED_SUPER) {
1629 if (!usb_endpoint_xfer_control(desc))
1630 maxburst = comp_desc->bMaxBurst;
1631
1632 if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc))
1633 esit = le16_to_cpu(comp_desc->wBytesPerInterval);
1634 } else if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
1635 (usb_endpoint_xfer_int(desc) ||
1636 usb_endpoint_xfer_isoc(desc))) {
1637 if (xudc->gadget.speed == USB_SPEED_HIGH) {
1638 maxburst = usb_endpoint_maxp_mult(desc) - 1;
1639 if (maxburst == 0x3) {
1640 dev_warn(xudc->dev,
1641 "invalid endpoint maxburst\n");
1642 maxburst = 0x2;
1643 }
1644 }
1645 esit = maxpacket * (maxburst + 1);
1646 }
1647
1648 memset(ep->context, 0, sizeof(*ep->context));
1649
1650 ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1651 ep_ctx_write_interval(ep->context, desc->bInterval);
1652 if (xudc->gadget.speed == USB_SPEED_SUPER) {
1653 if (usb_endpoint_xfer_isoc(desc)) {
1654 ep_ctx_write_mult(ep->context,
1655 comp_desc->bmAttributes & 0x3);
1656 }
1657
1658 if (usb_endpoint_xfer_bulk(desc)) {
1659 ep_ctx_write_max_pstreams(ep->context,
1660 comp_desc->bmAttributes &
1661 0x1f);
1662 ep_ctx_write_lsa(ep->context, 1);
1663 }
1664 }
1665
1666 if (!usb_endpoint_xfer_control(desc) && usb_endpoint_dir_out(desc))
1667 val = usb_endpoint_type(desc);
1668 else
1669 val = usb_endpoint_type(desc) + EP_TYPE_CONTROL;
1670
1671 ep_ctx_write_type(ep->context, val);
1672 ep_ctx_write_cerr(ep->context, 0x3);
1673 ep_ctx_write_max_packet_size(ep->context, maxpacket);
1674 ep_ctx_write_max_burst_size(ep->context, maxburst);
1675
1676 ep_ctx_write_deq_ptr(ep->context, ep->transfer_ring_phys);
1677 ep_ctx_write_dcs(ep->context, ep->pcs);
1678
1679 /* Select a reasonable average TRB length based on endpoint type. */
1680 switch (usb_endpoint_type(desc)) {
1681 case USB_ENDPOINT_XFER_CONTROL:
1682 val = 8;
1683 break;
1684 case USB_ENDPOINT_XFER_INT:
1685 val = 1024;
1686 break;
1687 case USB_ENDPOINT_XFER_BULK:
1688 case USB_ENDPOINT_XFER_ISOC:
1689 default:
1690 val = 3072;
1691 break;
1692 }
1693
1694 ep_ctx_write_avg_trb_len(ep->context, val);
1695 ep_ctx_write_max_esit_payload(ep->context, esit);
1696
1697 ep_ctx_write_cerrcnt(ep->context, 0x3);
1698}
1699
1700static void setup_link_trb(struct tegra_xudc_ep *ep,
1701 struct tegra_xudc_trb *trb)
1702{
1703 trb_write_data_ptr(trb, ep->transfer_ring_phys);
1704 trb_write_type(trb, TRB_TYPE_LINK);
1705 trb_write_toggle_cycle(trb, 1);
1706}
1707
1708static int __tegra_xudc_ep_disable(struct tegra_xudc_ep *ep)
1709{
1710 struct tegra_xudc *xudc = ep->xudc;
1711
1712 if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
1713 dev_err(xudc->dev, "endpoint %u already disabled\n",
1714 ep->index);
1715 return -EINVAL;
1716 }
1717
1718 ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1719
1720 ep_reload(xudc, ep->index);
1721
1722 tegra_xudc_ep_nuke(ep, -ESHUTDOWN);
1723
1724 xudc->nr_enabled_eps--;
1725 if (usb_endpoint_xfer_isoc(ep->desc))
1726 xudc->nr_isoch_eps--;
1727
1728 ep->desc = NULL;
1729 ep->comp_desc = NULL;
1730
1731 memset(ep->context, 0, sizeof(*ep->context));
1732
1733 ep_unpause(xudc, ep->index);
1734 ep_unhalt(xudc, ep->index);
1735 if (xudc_readl(xudc, EP_STOPPED) & BIT(ep->index))
1736 xudc_writel(xudc, BIT(ep->index), EP_STOPPED);
1737
1738 /*
1739 * If this is the last endpoint disabled in a de-configure request,
1740 * switch back to address state.
1741 */
1742 if ((xudc->device_state == USB_STATE_CONFIGURED) &&
1743 (xudc->nr_enabled_eps == 1)) {
1744 u32 val;
1745
1746 xudc->device_state = USB_STATE_ADDRESS;
1747 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1748
1749 val = xudc_readl(xudc, CTRL);
1750 val &= ~CTRL_RUN;
1751 xudc_writel(xudc, val, CTRL);
1752 }
1753
1754 dev_info(xudc->dev, "ep %u disabled\n", ep->index);
1755
1756 return 0;
1757}
1758
1759static int tegra_xudc_ep_disable(struct usb_ep *usb_ep)
1760{
1761 struct tegra_xudc_ep *ep;
1762 struct tegra_xudc *xudc;
1763 unsigned long flags;
1764 int ret;
1765
1766 if (!usb_ep)
1767 return -EINVAL;
1768
1769 ep = to_xudc_ep(usb_ep);
1770 xudc = ep->xudc;
1771
1772 spin_lock_irqsave(&xudc->lock, flags);
1773 if (xudc->powergated) {
1774 ret = -ESHUTDOWN;
1775 goto unlock;
1776 }
1777
1778 ret = __tegra_xudc_ep_disable(ep);
1779unlock:
1780 spin_unlock_irqrestore(&xudc->lock, flags);
1781
1782 return ret;
1783}
1784
1785static int __tegra_xudc_ep_enable(struct tegra_xudc_ep *ep,
1786 const struct usb_endpoint_descriptor *desc)
1787{
1788 struct tegra_xudc *xudc = ep->xudc;
1789 unsigned int i;
1790 u32 val;
1791
1792 if (xudc->gadget.speed == USB_SPEED_SUPER &&
1793 !usb_endpoint_xfer_control(desc) && !ep->usb_ep.comp_desc)
1794 return -EINVAL;
1795
1796 /* Disable the EP if it is not disabled */
1797 if (ep_ctx_read_state(ep->context) != EP_STATE_DISABLED)
1798 __tegra_xudc_ep_disable(ep);
1799
1800 ep->desc = desc;
1801 ep->comp_desc = ep->usb_ep.comp_desc;
1802
1803 if (usb_endpoint_xfer_isoc(desc)) {
1804 if (xudc->nr_isoch_eps > XUDC_MAX_ISOCH_EPS) {
1805 dev_err(xudc->dev, "too many isochronous endpoints\n");
1806 return -EBUSY;
1807 }
1808 xudc->nr_isoch_eps++;
1809 }
1810
1811 memset(ep->transfer_ring, 0, XUDC_TRANSFER_RING_SIZE *
1812 sizeof(*ep->transfer_ring));
1813 setup_link_trb(ep, &ep->transfer_ring[XUDC_TRANSFER_RING_SIZE - 1]);
1814
1815 ep->enq_ptr = 0;
1816 ep->deq_ptr = 0;
1817 ep->pcs = true;
1818 ep->ring_full = false;
1819 xudc->nr_enabled_eps++;
1820
1821 tegra_xudc_ep_context_setup(ep);
1822
1823 /*
1824 * No need to reload and un-halt EP0. This will be done automatically
1825 * once a valid SETUP packet is received.
1826 */
1827 if (usb_endpoint_xfer_control(desc))
1828 goto out;
1829
1830 /*
1831 * Transition to configured state once the first non-control
1832 * endpoint is enabled.
1833 */
1834 if (xudc->device_state == USB_STATE_ADDRESS) {
1835 val = xudc_readl(xudc, CTRL);
1836 val |= CTRL_RUN;
1837 xudc_writel(xudc, val, CTRL);
1838
1839 xudc->device_state = USB_STATE_CONFIGURED;
1840 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1841 }
1842
1843 if (usb_endpoint_xfer_isoc(desc)) {
1844 /*
1845 * Pause all bulk endpoints when enabling an isoch endpoint
1846 * to ensure the isoch endpoint is allocated enough bandwidth.
1847 */
1848 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1849 if (xudc->ep[i].desc &&
1850 usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1851 ep_pause(xudc, i);
1852 }
1853 }
1854
1855 ep_reload(xudc, ep->index);
1856 ep_unpause(xudc, ep->index);
1857 ep_unhalt(xudc, ep->index);
1858
1859 if (usb_endpoint_xfer_isoc(desc)) {
1860 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1861 if (xudc->ep[i].desc &&
1862 usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1863 ep_unpause(xudc, i);
1864 }
1865 }
1866
1867out:
1868 dev_info(xudc->dev, "EP %u (type: %s, dir: %s) enabled\n", ep->index,
1869 usb_ep_type_string(usb_endpoint_type(ep->desc)),
1870 usb_endpoint_dir_in(ep->desc) ? "in" : "out");
1871
1872 return 0;
1873}
1874
1875static int tegra_xudc_ep_enable(struct usb_ep *usb_ep,
1876 const struct usb_endpoint_descriptor *desc)
1877{
1878 struct tegra_xudc_ep *ep;
1879 struct tegra_xudc *xudc;
1880 unsigned long flags;
1881 int ret;
1882
1883 if (!usb_ep || !desc || (desc->bDescriptorType != USB_DT_ENDPOINT))
1884 return -EINVAL;
1885
1886 ep = to_xudc_ep(usb_ep);
1887 xudc = ep->xudc;
1888
1889 spin_lock_irqsave(&xudc->lock, flags);
1890 if (xudc->powergated) {
1891 ret = -ESHUTDOWN;
1892 goto unlock;
1893 }
1894
1895 ret = __tegra_xudc_ep_enable(ep, desc);
1896unlock:
1897 spin_unlock_irqrestore(&xudc->lock, flags);
1898
1899 return ret;
1900}
1901
1902static struct usb_request *
1903tegra_xudc_ep_alloc_request(struct usb_ep *usb_ep, gfp_t gfp)
1904{
1905 struct tegra_xudc_request *req;
1906
1907 req = kzalloc(sizeof(*req), gfp);
1908 if (!req)
1909 return NULL;
1910
1911 INIT_LIST_HEAD(&req->list);
1912
1913 return &req->usb_req;
1914}
1915
1916static void tegra_xudc_ep_free_request(struct usb_ep *usb_ep,
1917 struct usb_request *usb_req)
1918{
1919 struct tegra_xudc_request *req = to_xudc_req(usb_req);
1920
1921 kfree(req);
1922}
1923
1924static const struct usb_ep_ops tegra_xudc_ep_ops = {
1925 .enable = tegra_xudc_ep_enable,
1926 .disable = tegra_xudc_ep_disable,
1927 .alloc_request = tegra_xudc_ep_alloc_request,
1928 .free_request = tegra_xudc_ep_free_request,
1929 .queue = tegra_xudc_ep_queue,
1930 .dequeue = tegra_xudc_ep_dequeue,
1931 .set_halt = tegra_xudc_ep_set_halt,
1932};
1933
1934static int tegra_xudc_ep0_enable(struct usb_ep *usb_ep,
1935 const struct usb_endpoint_descriptor *desc)
1936{
1937 return -EBUSY;
1938}
1939
1940static int tegra_xudc_ep0_disable(struct usb_ep *usb_ep)
1941{
1942 return -EBUSY;
1943}
1944
1945static const struct usb_ep_ops tegra_xudc_ep0_ops = {
1946 .enable = tegra_xudc_ep0_enable,
1947 .disable = tegra_xudc_ep0_disable,
1948 .alloc_request = tegra_xudc_ep_alloc_request,
1949 .free_request = tegra_xudc_ep_free_request,
1950 .queue = tegra_xudc_ep_queue,
1951 .dequeue = tegra_xudc_ep_dequeue,
1952 .set_halt = tegra_xudc_ep_set_halt,
1953};
1954
1955static int tegra_xudc_gadget_get_frame(struct usb_gadget *gadget)
1956{
1957 struct tegra_xudc *xudc = to_xudc(gadget);
1958 unsigned long flags;
1959 int ret;
1960
1961 spin_lock_irqsave(&xudc->lock, flags);
1962 if (xudc->powergated) {
1963 ret = -ESHUTDOWN;
1964 goto unlock;
1965 }
1966
1967 ret = (xudc_readl(xudc, MFINDEX) & MFINDEX_FRAME_MASK) >>
1968 MFINDEX_FRAME_SHIFT;
1969unlock:
1970 spin_unlock_irqrestore(&xudc->lock, flags);
1971
1972 return ret;
1973}
1974
1975static void tegra_xudc_resume_device_state(struct tegra_xudc *xudc)
1976{
1977 unsigned int i;
1978 u32 val;
1979
1980 ep_unpause_all(xudc);
1981
1982 /* Direct link to U0. */
1983 val = xudc_readl(xudc, PORTSC);
1984 if (((val & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT) != PORTSC_PLS_U0) {
1985 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
1986 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
1987 xudc_writel(xudc, val, PORTSC);
1988 }
1989
1990 if (xudc->device_state == USB_STATE_SUSPENDED) {
1991 xudc->device_state = xudc->resume_state;
1992 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1993 xudc->resume_state = 0;
1994 }
1995
1996 /*
1997 * Doorbells may be dropped if they are sent too soon (< ~200ns)
1998 * after unpausing the endpoint. Wait for 500ns just to be safe.
1999 */
2000 ndelay(500);
2001 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2002 tegra_xudc_ep_ring_doorbell(&xudc->ep[i]);
2003}
2004
2005static int tegra_xudc_gadget_wakeup(struct usb_gadget *gadget)
2006{
2007 struct tegra_xudc *xudc = to_xudc(gadget);
2008 unsigned long flags;
2009 int ret = 0;
2010 u32 val;
2011
2012 spin_lock_irqsave(&xudc->lock, flags);
2013
2014 if (xudc->powergated) {
2015 ret = -ESHUTDOWN;
2016 goto unlock;
2017 }
2018 val = xudc_readl(xudc, PORTPM);
2019 dev_dbg(xudc->dev, "%s: PORTPM=%#x, speed=%x\n", __func__,
2020 val, gadget->speed);
2021
2022 if (((xudc->gadget.speed <= USB_SPEED_HIGH) &&
2023 (val & PORTPM_RWE)) ||
2024 ((xudc->gadget.speed == USB_SPEED_SUPER) &&
2025 (val & PORTPM_FRWE))) {
2026 tegra_xudc_resume_device_state(xudc);
2027
2028 /* Send Device Notification packet. */
2029 if (xudc->gadget.speed == USB_SPEED_SUPER) {
2030 val = DEVNOTIF_LO_TYPE(DEVNOTIF_LO_TYPE_FUNCTION_WAKE)
2031 | DEVNOTIF_LO_TRIG;
2032 xudc_writel(xudc, 0, DEVNOTIF_HI);
2033 xudc_writel(xudc, val, DEVNOTIF_LO);
2034 }
2035 }
2036
2037unlock:
2038 dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2039 spin_unlock_irqrestore(&xudc->lock, flags);
2040
2041 return ret;
2042}
2043
2044static int tegra_xudc_gadget_pullup(struct usb_gadget *gadget, int is_on)
2045{
2046 struct tegra_xudc *xudc = to_xudc(gadget);
2047 unsigned long flags;
2048 u32 val;
2049
2050 pm_runtime_get_sync(xudc->dev);
2051
2052 spin_lock_irqsave(&xudc->lock, flags);
2053
2054 if (is_on != xudc->pullup) {
2055 val = xudc_readl(xudc, CTRL);
2056 if (is_on)
2057 val |= CTRL_ENABLE;
2058 else
2059 val &= ~CTRL_ENABLE;
2060 xudc_writel(xudc, val, CTRL);
2061 }
2062
2063 xudc->pullup = is_on;
2064 dev_dbg(xudc->dev, "%s: pullup:%d", __func__, is_on);
2065
2066 spin_unlock_irqrestore(&xudc->lock, flags);
2067
2068 pm_runtime_put(xudc->dev);
2069
2070 return 0;
2071}
2072
2073static int tegra_xudc_gadget_start(struct usb_gadget *gadget,
2074 struct usb_gadget_driver *driver)
2075{
2076 struct tegra_xudc *xudc = to_xudc(gadget);
2077 unsigned long flags;
2078 u32 val;
2079 int ret;
2080 unsigned int i;
2081
2082 if (!driver)
2083 return -EINVAL;
2084
2085 pm_runtime_get_sync(xudc->dev);
2086
2087 spin_lock_irqsave(&xudc->lock, flags);
2088
2089 if (xudc->driver) {
2090 ret = -EBUSY;
2091 goto unlock;
2092 }
2093
2094 xudc->setup_state = WAIT_FOR_SETUP;
2095 xudc->device_state = USB_STATE_DEFAULT;
2096 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2097
2098 ret = __tegra_xudc_ep_enable(&xudc->ep[0], &tegra_xudc_ep0_desc);
2099 if (ret < 0)
2100 goto unlock;
2101
2102 val = xudc_readl(xudc, CTRL);
2103 val |= CTRL_IE | CTRL_LSE;
2104 xudc_writel(xudc, val, CTRL);
2105
2106 val = xudc_readl(xudc, PORTHALT);
2107 val |= PORTHALT_STCHG_INTR_EN;
2108 xudc_writel(xudc, val, PORTHALT);
2109
2110 if (xudc->pullup) {
2111 val = xudc_readl(xudc, CTRL);
2112 val |= CTRL_ENABLE;
2113 xudc_writel(xudc, val, CTRL);
2114 }
2115
2116 for (i = 0; i < xudc->soc->num_phys; i++)
2117 if (xudc->usbphy[i])
2118 otg_set_peripheral(xudc->usbphy[i]->otg, gadget);
2119
2120 xudc->driver = driver;
2121unlock:
2122 dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2123 spin_unlock_irqrestore(&xudc->lock, flags);
2124
2125 pm_runtime_put(xudc->dev);
2126
2127 return ret;
2128}
2129
2130static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
2131{
2132 struct tegra_xudc *xudc = to_xudc(gadget);
2133 unsigned long flags;
2134 u32 val;
2135 unsigned int i;
2136
2137 pm_runtime_get_sync(xudc->dev);
2138
2139 spin_lock_irqsave(&xudc->lock, flags);
2140
2141 for (i = 0; i < xudc->soc->num_phys; i++)
2142 if (xudc->usbphy[i])
2143 otg_set_peripheral(xudc->usbphy[i]->otg, NULL);
2144
2145 val = xudc_readl(xudc, CTRL);
2146 val &= ~(CTRL_IE | CTRL_ENABLE);
2147 xudc_writel(xudc, val, CTRL);
2148
2149 __tegra_xudc_ep_disable(&xudc->ep[0]);
2150
2151 xudc->driver = NULL;
2152 dev_dbg(xudc->dev, "Gadget stopped");
2153
2154 spin_unlock_irqrestore(&xudc->lock, flags);
2155
2156 pm_runtime_put(xudc->dev);
2157
2158 return 0;
2159}
2160
2161static int tegra_xudc_gadget_vbus_draw(struct usb_gadget *gadget,
2162 unsigned int m_a)
2163{
2164 struct tegra_xudc *xudc = to_xudc(gadget);
2165
2166 dev_dbg(xudc->dev, "%s: %u mA\n", __func__, m_a);
2167
2168 if (xudc->curr_usbphy && xudc->curr_usbphy->chg_type == SDP_TYPE)
2169 return usb_phy_set_power(xudc->curr_usbphy, m_a);
2170
2171 return 0;
2172}
2173
2174static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
2175{
2176 struct tegra_xudc *xudc = to_xudc(gadget);
2177
2178 dev_dbg(xudc->dev, "%s: %d\n", __func__, is_on);
2179 xudc->selfpowered = !!is_on;
2180
2181 return 0;
2182}
2183
2184static const struct usb_gadget_ops tegra_xudc_gadget_ops = {
2185 .get_frame = tegra_xudc_gadget_get_frame,
2186 .wakeup = tegra_xudc_gadget_wakeup,
2187 .pullup = tegra_xudc_gadget_pullup,
2188 .udc_start = tegra_xudc_gadget_start,
2189 .udc_stop = tegra_xudc_gadget_stop,
2190 .vbus_draw = tegra_xudc_gadget_vbus_draw,
2191 .set_selfpowered = tegra_xudc_set_selfpowered,
2192};
2193
2194static void no_op_complete(struct usb_ep *ep, struct usb_request *req)
2195{
2196}
2197
2198static int
2199tegra_xudc_ep0_queue_status(struct tegra_xudc *xudc,
2200 void (*cmpl)(struct usb_ep *, struct usb_request *))
2201{
2202 xudc->ep0_req->usb_req.buf = NULL;
2203 xudc->ep0_req->usb_req.dma = 0;
2204 xudc->ep0_req->usb_req.length = 0;
2205 xudc->ep0_req->usb_req.complete = cmpl;
2206 xudc->ep0_req->usb_req.context = xudc;
2207
2208 return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2209}
2210
2211static int
2212tegra_xudc_ep0_queue_data(struct tegra_xudc *xudc, void *buf, size_t len,
2213 void (*cmpl)(struct usb_ep *, struct usb_request *))
2214{
2215 xudc->ep0_req->usb_req.buf = buf;
2216 xudc->ep0_req->usb_req.length = len;
2217 xudc->ep0_req->usb_req.complete = cmpl;
2218 xudc->ep0_req->usb_req.context = xudc;
2219
2220 return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2221}
2222
2223static void tegra_xudc_ep0_req_done(struct tegra_xudc *xudc)
2224{
2225 switch (xudc->setup_state) {
2226 case DATA_STAGE_XFER:
2227 xudc->setup_state = STATUS_STAGE_RECV;
2228 tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2229 break;
2230 case DATA_STAGE_RECV:
2231 xudc->setup_state = STATUS_STAGE_XFER;
2232 tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2233 break;
2234 default:
2235 xudc->setup_state = WAIT_FOR_SETUP;
2236 break;
2237 }
2238}
2239
2240static int tegra_xudc_ep0_delegate_req(struct tegra_xudc *xudc,
2241 struct usb_ctrlrequest *ctrl)
2242{
2243 int ret;
2244
2245 spin_unlock(&xudc->lock);
2246 ret = xudc->driver->setup(&xudc->gadget, ctrl);
2247 spin_lock(&xudc->lock);
2248
2249 return ret;
2250}
2251
2252static void set_feature_complete(struct usb_ep *ep, struct usb_request *req)
2253{
2254 struct tegra_xudc *xudc = req->context;
2255
2256 if (xudc->test_mode_pattern) {
2257 xudc_writel(xudc, xudc->test_mode_pattern, PORT_TM);
2258 xudc->test_mode_pattern = 0;
2259 }
2260}
2261
2262static int tegra_xudc_ep0_set_feature(struct tegra_xudc *xudc,
2263 struct usb_ctrlrequest *ctrl)
2264{
2265 bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
2266 u32 feature = le16_to_cpu(ctrl->wValue);
2267 u32 index = le16_to_cpu(ctrl->wIndex);
2268 u32 val, ep;
2269 int ret;
2270
2271 if (le16_to_cpu(ctrl->wLength) != 0)
2272 return -EINVAL;
2273
2274 switch (ctrl->bRequestType & USB_RECIP_MASK) {
2275 case USB_RECIP_DEVICE:
2276 switch (feature) {
2277 case USB_DEVICE_REMOTE_WAKEUP:
2278 if ((xudc->gadget.speed == USB_SPEED_SUPER) ||
2279 (xudc->device_state == USB_STATE_DEFAULT))
2280 return -EINVAL;
2281
2282 val = xudc_readl(xudc, PORTPM);
2283 if (set)
2284 val |= PORTPM_RWE;
2285 else
2286 val &= ~PORTPM_RWE;
2287
2288 xudc_writel(xudc, val, PORTPM);
2289 break;
2290 case USB_DEVICE_U1_ENABLE:
2291 case USB_DEVICE_U2_ENABLE:
2292 if ((xudc->device_state != USB_STATE_CONFIGURED) ||
2293 (xudc->gadget.speed != USB_SPEED_SUPER))
2294 return -EINVAL;
2295
2296 val = xudc_readl(xudc, PORTPM);
2297 if ((feature == USB_DEVICE_U1_ENABLE) &&
2298 xudc->soc->u1_enable) {
2299 if (set)
2300 val |= PORTPM_U1E;
2301 else
2302 val &= ~PORTPM_U1E;
2303 }
2304
2305 if ((feature == USB_DEVICE_U2_ENABLE) &&
2306 xudc->soc->u2_enable) {
2307 if (set)
2308 val |= PORTPM_U2E;
2309 else
2310 val &= ~PORTPM_U2E;
2311 }
2312
2313 xudc_writel(xudc, val, PORTPM);
2314 break;
2315 case USB_DEVICE_TEST_MODE:
2316 if (xudc->gadget.speed != USB_SPEED_HIGH)
2317 return -EINVAL;
2318
2319 if (!set)
2320 return -EINVAL;
2321
2322 xudc->test_mode_pattern = index >> 8;
2323 break;
2324 default:
2325 return -EINVAL;
2326 }
2327
2328 break;
2329 case USB_RECIP_INTERFACE:
2330 if (xudc->device_state != USB_STATE_CONFIGURED)
2331 return -EINVAL;
2332
2333 switch (feature) {
2334 case USB_INTRF_FUNC_SUSPEND:
2335 if (set) {
2336 val = xudc_readl(xudc, PORTPM);
2337
2338 if (index & USB_INTRF_FUNC_SUSPEND_RW)
2339 val |= PORTPM_FRWE;
2340 else
2341 val &= ~PORTPM_FRWE;
2342
2343 xudc_writel(xudc, val, PORTPM);
2344 }
2345
2346 return tegra_xudc_ep0_delegate_req(xudc, ctrl);
2347 default:
2348 return -EINVAL;
2349 }
2350
2351 break;
2352 case USB_RECIP_ENDPOINT:
2353 ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2354 ((index & USB_DIR_IN) ? 1 : 0);
2355
2356 if ((xudc->device_state == USB_STATE_DEFAULT) ||
2357 ((xudc->device_state == USB_STATE_ADDRESS) &&
2358 (index != 0)))
2359 return -EINVAL;
2360
2361 ret = __tegra_xudc_ep_set_halt(&xudc->ep[ep], set);
2362 if (ret < 0)
2363 return ret;
2364 break;
2365 default:
2366 return -EINVAL;
2367 }
2368
2369 return tegra_xudc_ep0_queue_status(xudc, set_feature_complete);
2370}
2371
2372static int tegra_xudc_ep0_get_status(struct tegra_xudc *xudc,
2373 struct usb_ctrlrequest *ctrl)
2374{
2375 struct tegra_xudc_ep_context *ep_ctx;
2376 u32 val, ep, index = le16_to_cpu(ctrl->wIndex);
2377 u16 status = 0;
2378
2379 if (!(ctrl->bRequestType & USB_DIR_IN))
2380 return -EINVAL;
2381
2382 if ((le16_to_cpu(ctrl->wValue) != 0) ||
2383 (le16_to_cpu(ctrl->wLength) != 2))
2384 return -EINVAL;
2385
2386 switch (ctrl->bRequestType & USB_RECIP_MASK) {
2387 case USB_RECIP_DEVICE:
2388 val = xudc_readl(xudc, PORTPM);
2389
2390 if (xudc->selfpowered)
2391 status |= BIT(USB_DEVICE_SELF_POWERED);
2392
2393 if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
2394 (val & PORTPM_RWE))
2395 status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
2396
2397 if (xudc->gadget.speed == USB_SPEED_SUPER) {
2398 if (val & PORTPM_U1E)
2399 status |= BIT(USB_DEV_STAT_U1_ENABLED);
2400 if (val & PORTPM_U2E)
2401 status |= BIT(USB_DEV_STAT_U2_ENABLED);
2402 }
2403 break;
2404 case USB_RECIP_INTERFACE:
2405 if (xudc->gadget.speed == USB_SPEED_SUPER) {
2406 status |= USB_INTRF_STAT_FUNC_RW_CAP;
2407 val = xudc_readl(xudc, PORTPM);
2408 if (val & PORTPM_FRWE)
2409 status |= USB_INTRF_STAT_FUNC_RW;
2410 }
2411 break;
2412 case USB_RECIP_ENDPOINT:
2413 ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2414 ((index & USB_DIR_IN) ? 1 : 0);
2415 ep_ctx = &xudc->ep_context[ep];
2416
2417 if ((xudc->device_state != USB_STATE_CONFIGURED) &&
2418 ((xudc->device_state != USB_STATE_ADDRESS) || (ep != 0)))
2419 return -EINVAL;
2420
2421 if (ep_ctx_read_state(ep_ctx) == EP_STATE_DISABLED)
2422 return -EINVAL;
2423
2424 if (xudc_readl(xudc, EP_HALT) & BIT(ep))
2425 status |= BIT(USB_ENDPOINT_HALT);
2426 break;
2427 default:
2428 return -EINVAL;
2429 }
2430
2431 xudc->status_buf = cpu_to_le16(status);
2432 return tegra_xudc_ep0_queue_data(xudc, &xudc->status_buf,
2433 sizeof(xudc->status_buf),
2434 no_op_complete);
2435}
2436
2437static void set_sel_complete(struct usb_ep *ep, struct usb_request *req)
2438{
2439 /* Nothing to do with SEL values */
2440}
2441
2442static int tegra_xudc_ep0_set_sel(struct tegra_xudc *xudc,
2443 struct usb_ctrlrequest *ctrl)
2444{
2445 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2446 USB_TYPE_STANDARD))
2447 return -EINVAL;
2448
2449 if (xudc->device_state == USB_STATE_DEFAULT)
2450 return -EINVAL;
2451
2452 if ((le16_to_cpu(ctrl->wIndex) != 0) ||
2453 (le16_to_cpu(ctrl->wValue) != 0) ||
2454 (le16_to_cpu(ctrl->wLength) != 6))
2455 return -EINVAL;
2456
2457 return tegra_xudc_ep0_queue_data(xudc, &xudc->sel_timing,
2458 sizeof(xudc->sel_timing),
2459 set_sel_complete);
2460}
2461
2462static void set_isoch_delay_complete(struct usb_ep *ep, struct usb_request *req)
2463{
2464 /* Nothing to do with isoch delay */
2465}
2466
2467static int tegra_xudc_ep0_set_isoch_delay(struct tegra_xudc *xudc,
2468 struct usb_ctrlrequest *ctrl)
2469{
2470 u32 delay = le16_to_cpu(ctrl->wValue);
2471
2472 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2473 USB_TYPE_STANDARD))
2474 return -EINVAL;
2475
2476 if ((delay > 65535) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2477 (le16_to_cpu(ctrl->wLength) != 0))
2478 return -EINVAL;
2479
2480 xudc->isoch_delay = delay;
2481
2482 return tegra_xudc_ep0_queue_status(xudc, set_isoch_delay_complete);
2483}
2484
2485static void set_address_complete(struct usb_ep *ep, struct usb_request *req)
2486{
2487 struct tegra_xudc *xudc = req->context;
2488
2489 if ((xudc->device_state == USB_STATE_DEFAULT) &&
2490 (xudc->dev_addr != 0)) {
2491 xudc->device_state = USB_STATE_ADDRESS;
2492 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2493 } else if ((xudc->device_state == USB_STATE_ADDRESS) &&
2494 (xudc->dev_addr == 0)) {
2495 xudc->device_state = USB_STATE_DEFAULT;
2496 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2497 }
2498}
2499
2500static int tegra_xudc_ep0_set_address(struct tegra_xudc *xudc,
2501 struct usb_ctrlrequest *ctrl)
2502{
2503 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2504 u32 val, addr = le16_to_cpu(ctrl->wValue);
2505
2506 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2507 USB_TYPE_STANDARD))
2508 return -EINVAL;
2509
2510 if ((addr > 127) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2511 (le16_to_cpu(ctrl->wLength) != 0))
2512 return -EINVAL;
2513
2514 if (xudc->device_state == USB_STATE_CONFIGURED)
2515 return -EINVAL;
2516
2517 dev_dbg(xudc->dev, "set address: %u\n", addr);
2518
2519 xudc->dev_addr = addr;
2520 val = xudc_readl(xudc, CTRL);
2521 val &= ~(CTRL_DEVADDR_MASK);
2522 val |= CTRL_DEVADDR(addr);
2523 xudc_writel(xudc, val, CTRL);
2524
2525 ep_ctx_write_devaddr(ep0->context, addr);
2526
2527 return tegra_xudc_ep0_queue_status(xudc, set_address_complete);
2528}
2529
2530static int tegra_xudc_ep0_standard_req(struct tegra_xudc *xudc,
2531 struct usb_ctrlrequest *ctrl)
2532{
2533 int ret;
2534
2535 switch (ctrl->bRequest) {
2536 case USB_REQ_GET_STATUS:
2537 dev_dbg(xudc->dev, "USB_REQ_GET_STATUS\n");
2538 ret = tegra_xudc_ep0_get_status(xudc, ctrl);
2539 break;
2540 case USB_REQ_SET_ADDRESS:
2541 dev_dbg(xudc->dev, "USB_REQ_SET_ADDRESS\n");
2542 ret = tegra_xudc_ep0_set_address(xudc, ctrl);
2543 break;
2544 case USB_REQ_SET_SEL:
2545 dev_dbg(xudc->dev, "USB_REQ_SET_SEL\n");
2546 ret = tegra_xudc_ep0_set_sel(xudc, ctrl);
2547 break;
2548 case USB_REQ_SET_ISOCH_DELAY:
2549 dev_dbg(xudc->dev, "USB_REQ_SET_ISOCH_DELAY\n");
2550 ret = tegra_xudc_ep0_set_isoch_delay(xudc, ctrl);
2551 break;
2552 case USB_REQ_CLEAR_FEATURE:
2553 case USB_REQ_SET_FEATURE:
2554 dev_dbg(xudc->dev, "USB_REQ_CLEAR/SET_FEATURE\n");
2555 ret = tegra_xudc_ep0_set_feature(xudc, ctrl);
2556 break;
2557 case USB_REQ_SET_CONFIGURATION:
2558 dev_dbg(xudc->dev, "USB_REQ_SET_CONFIGURATION\n");
2559 /*
2560 * In theory we need to clear RUN bit before status stage of
2561 * deconfig request sent, but this seems to be causing problems.
2562 * Clear RUN once all endpoints are disabled instead.
2563 */
2564 fallthrough;
2565 default:
2566 ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2567 break;
2568 }
2569
2570 return ret;
2571}
2572
2573static void tegra_xudc_handle_ep0_setup_packet(struct tegra_xudc *xudc,
2574 struct usb_ctrlrequest *ctrl,
2575 u16 seq_num)
2576{
2577 int ret;
2578
2579 xudc->setup_seq_num = seq_num;
2580
2581 /* Ensure EP0 is unhalted. */
2582 ep_unhalt(xudc, 0);
2583
2584 /*
2585 * On Tegra210, setup packets with sequence numbers 0xfffe or 0xffff
2586 * are invalid. Halt EP0 until we get a valid packet.
2587 */
2588 if (xudc->soc->invalid_seq_num &&
2589 (seq_num == 0xfffe || seq_num == 0xffff)) {
2590 dev_warn(xudc->dev, "invalid sequence number detected\n");
2591 ep_halt(xudc, 0);
2592 return;
2593 }
2594
2595 if (ctrl->wLength)
2596 xudc->setup_state = (ctrl->bRequestType & USB_DIR_IN) ?
2597 DATA_STAGE_XFER : DATA_STAGE_RECV;
2598 else
2599 xudc->setup_state = STATUS_STAGE_XFER;
2600
2601 if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
2602 ret = tegra_xudc_ep0_standard_req(xudc, ctrl);
2603 else
2604 ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2605
2606 if (ret < 0) {
2607 dev_warn(xudc->dev, "setup request failed: %d\n", ret);
2608 xudc->setup_state = WAIT_FOR_SETUP;
2609 ep_halt(xudc, 0);
2610 }
2611}
2612
2613static void tegra_xudc_handle_ep0_event(struct tegra_xudc *xudc,
2614 struct tegra_xudc_trb *event)
2615{
2616 struct usb_ctrlrequest *ctrl = (struct usb_ctrlrequest *)event;
2617 u16 seq_num = trb_read_seq_num(event);
2618
2619 if (xudc->setup_state != WAIT_FOR_SETUP) {
2620 /*
2621 * The controller is in the process of handling another
2622 * setup request. Queue subsequent requests and handle
2623 * the last one once the controller reports a sequence
2624 * number error.
2625 */
2626 memcpy(&xudc->setup_packet.ctrl_req, ctrl, sizeof(*ctrl));
2627 xudc->setup_packet.seq_num = seq_num;
2628 xudc->queued_setup_packet = true;
2629 } else {
2630 tegra_xudc_handle_ep0_setup_packet(xudc, ctrl, seq_num);
2631 }
2632}
2633
2634static struct tegra_xudc_request *
2635trb_to_request(struct tegra_xudc_ep *ep, struct tegra_xudc_trb *trb)
2636{
2637 struct tegra_xudc_request *req;
2638
2639 list_for_each_entry(req, &ep->queue, list) {
2640 if (!req->trbs_queued)
2641 break;
2642
2643 if (trb_in_request(ep, req, trb))
2644 return req;
2645 }
2646
2647 return NULL;
2648}
2649
2650static void tegra_xudc_handle_transfer_completion(struct tegra_xudc *xudc,
2651 struct tegra_xudc_ep *ep,
2652 struct tegra_xudc_trb *event)
2653{
2654 struct tegra_xudc_request *req;
2655 struct tegra_xudc_trb *trb;
2656 bool short_packet;
2657
2658 short_packet = (trb_read_cmpl_code(event) ==
2659 TRB_CMPL_CODE_SHORT_PACKET);
2660
2661 trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2662 req = trb_to_request(ep, trb);
2663
2664 /*
2665 * TDs are complete on short packet or when the completed TRB is the
2666 * last TRB in the TD (the CHAIN bit is unset).
2667 */
2668 if (req && (short_packet || (!trb_read_chain(trb) &&
2669 (req->trbs_needed == req->trbs_queued)))) {
2670 struct tegra_xudc_trb *last = req->last_trb;
2671 unsigned int residual;
2672
2673 residual = trb_read_transfer_len(event);
2674 req->usb_req.actual = req->usb_req.length - residual;
2675
2676 dev_dbg(xudc->dev, "bytes transferred %u / %u\n",
2677 req->usb_req.actual, req->usb_req.length);
2678
2679 tegra_xudc_req_done(ep, req, 0);
2680
2681 if (ep->desc && usb_endpoint_xfer_control(ep->desc))
2682 tegra_xudc_ep0_req_done(xudc);
2683
2684 /*
2685 * Advance the dequeue pointer past the end of the current TD
2686 * on short packet completion.
2687 */
2688 if (short_packet) {
2689 ep->deq_ptr = (last - ep->transfer_ring) + 1;
2690 if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2691 ep->deq_ptr = 0;
2692 }
2693 } else if (!req) {
2694 dev_warn(xudc->dev, "transfer event on dequeued request\n");
2695 }
2696
2697 if (ep->desc)
2698 tegra_xudc_ep_kick_queue(ep);
2699}
2700
2701static void tegra_xudc_handle_transfer_event(struct tegra_xudc *xudc,
2702 struct tegra_xudc_trb *event)
2703{
2704 unsigned int ep_index = trb_read_endpoint_id(event);
2705 struct tegra_xudc_ep *ep = &xudc->ep[ep_index];
2706 struct tegra_xudc_trb *trb;
2707 u16 comp_code;
2708
2709 if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
2710 dev_warn(xudc->dev, "transfer event on disabled EP %u\n",
2711 ep_index);
2712 return;
2713 }
2714
2715 /* Update transfer ring dequeue pointer. */
2716 trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2717 comp_code = trb_read_cmpl_code(event);
2718 if (comp_code != TRB_CMPL_CODE_BABBLE_DETECTED_ERR) {
2719 ep->deq_ptr = (trb - ep->transfer_ring) + 1;
2720
2721 if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2722 ep->deq_ptr = 0;
2723 ep->ring_full = false;
2724 }
2725
2726 switch (comp_code) {
2727 case TRB_CMPL_CODE_SUCCESS:
2728 case TRB_CMPL_CODE_SHORT_PACKET:
2729 tegra_xudc_handle_transfer_completion(xudc, ep, event);
2730 break;
2731 case TRB_CMPL_CODE_HOST_REJECTED:
2732 dev_info(xudc->dev, "stream rejected on EP %u\n", ep_index);
2733
2734 ep->stream_rejected = true;
2735 break;
2736 case TRB_CMPL_CODE_PRIME_PIPE_RECEIVED:
2737 dev_info(xudc->dev, "prime pipe received on EP %u\n", ep_index);
2738
2739 if (ep->stream_rejected) {
2740 ep->stream_rejected = false;
2741 /*
2742 * An EP is stopped when a stream is rejected. Wait
2743 * for the EP to report that it is stopped and then
2744 * un-stop it.
2745 */
2746 ep_wait_for_stopped(xudc, ep_index);
2747 }
2748 tegra_xudc_ep_ring_doorbell(ep);
2749 break;
2750 case TRB_CMPL_CODE_BABBLE_DETECTED_ERR:
2751 /*
2752 * Wait for the EP to be stopped so the controller stops
2753 * processing doorbells.
2754 */
2755 ep_wait_for_stopped(xudc, ep_index);
2756 ep->enq_ptr = ep->deq_ptr;
2757 tegra_xudc_ep_nuke(ep, -EIO);
2758 fallthrough;
2759 case TRB_CMPL_CODE_STREAM_NUMP_ERROR:
2760 case TRB_CMPL_CODE_CTRL_DIR_ERR:
2761 case TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR:
2762 case TRB_CMPL_CODE_RING_UNDERRUN:
2763 case TRB_CMPL_CODE_RING_OVERRUN:
2764 case TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN:
2765 case TRB_CMPL_CODE_USB_TRANS_ERR:
2766 case TRB_CMPL_CODE_TRB_ERR:
2767 dev_err(xudc->dev, "completion error %#x on EP %u\n",
2768 comp_code, ep_index);
2769
2770 ep_halt(xudc, ep_index);
2771 break;
2772 case TRB_CMPL_CODE_CTRL_SEQNUM_ERR:
2773 dev_info(xudc->dev, "sequence number error\n");
2774
2775 /*
2776 * Kill any queued control request and skip to the last
2777 * setup packet we received.
2778 */
2779 tegra_xudc_ep_nuke(ep, -EINVAL);
2780 xudc->setup_state = WAIT_FOR_SETUP;
2781 if (!xudc->queued_setup_packet)
2782 break;
2783
2784 tegra_xudc_handle_ep0_setup_packet(xudc,
2785 &xudc->setup_packet.ctrl_req,
2786 xudc->setup_packet.seq_num);
2787 xudc->queued_setup_packet = false;
2788 break;
2789 case TRB_CMPL_CODE_STOPPED:
2790 dev_dbg(xudc->dev, "stop completion code on EP %u\n",
2791 ep_index);
2792
2793 /* Disconnected. */
2794 tegra_xudc_ep_nuke(ep, -ECONNREFUSED);
2795 break;
2796 default:
2797 dev_dbg(xudc->dev, "completion event %#x on EP %u\n",
2798 comp_code, ep_index);
2799 break;
2800 }
2801}
2802
2803static void tegra_xudc_reset(struct tegra_xudc *xudc)
2804{
2805 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2806 dma_addr_t deq_ptr;
2807 unsigned int i;
2808
2809 xudc->setup_state = WAIT_FOR_SETUP;
2810 xudc->device_state = USB_STATE_DEFAULT;
2811 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2812
2813 ep_unpause_all(xudc);
2814
2815 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2816 tegra_xudc_ep_nuke(&xudc->ep[i], -ESHUTDOWN);
2817
2818 /*
2819 * Reset sequence number and dequeue pointer to flush the transfer
2820 * ring.
2821 */
2822 ep0->deq_ptr = ep0->enq_ptr;
2823 ep0->ring_full = false;
2824
2825 xudc->setup_seq_num = 0;
2826 xudc->queued_setup_packet = false;
2827
2828 ep_ctx_write_rsvd(ep0->context, 0);
2829 ep_ctx_write_partial_td(ep0->context, 0);
2830 ep_ctx_write_splitxstate(ep0->context, 0);
2831 ep_ctx_write_seq_num(ep0->context, 0);
2832
2833 deq_ptr = trb_virt_to_phys(ep0, &ep0->transfer_ring[ep0->deq_ptr]);
2834
2835 if (!dma_mapping_error(xudc->dev, deq_ptr)) {
2836 ep_ctx_write_deq_ptr(ep0->context, deq_ptr);
2837 ep_ctx_write_dcs(ep0->context, ep0->pcs);
2838 }
2839
2840 ep_unhalt_all(xudc);
2841 ep_reload(xudc, 0);
2842 ep_unpause(xudc, 0);
2843}
2844
2845static void tegra_xudc_port_connect(struct tegra_xudc *xudc)
2846{
2847 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2848 u16 maxpacket;
2849 u32 val;
2850
2851 val = (xudc_readl(xudc, PORTSC) & PORTSC_PS_MASK) >> PORTSC_PS_SHIFT;
2852 switch (val) {
2853 case PORTSC_PS_LS:
2854 xudc->gadget.speed = USB_SPEED_LOW;
2855 break;
2856 case PORTSC_PS_FS:
2857 xudc->gadget.speed = USB_SPEED_FULL;
2858 break;
2859 case PORTSC_PS_HS:
2860 xudc->gadget.speed = USB_SPEED_HIGH;
2861 break;
2862 case PORTSC_PS_SS:
2863 xudc->gadget.speed = USB_SPEED_SUPER;
2864 break;
2865 default:
2866 xudc->gadget.speed = USB_SPEED_UNKNOWN;
2867 break;
2868 }
2869
2870 xudc->device_state = USB_STATE_DEFAULT;
2871 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2872
2873 xudc->setup_state = WAIT_FOR_SETUP;
2874
2875 if (xudc->gadget.speed == USB_SPEED_SUPER)
2876 maxpacket = 512;
2877 else
2878 maxpacket = 64;
2879
2880 ep_ctx_write_max_packet_size(ep0->context, maxpacket);
2881 tegra_xudc_ep0_desc.wMaxPacketSize = cpu_to_le16(maxpacket);
2882 usb_ep_set_maxpacket_limit(&ep0->usb_ep, maxpacket);
2883
2884 if (!xudc->soc->u1_enable) {
2885 val = xudc_readl(xudc, PORTPM);
2886 val &= ~(PORTPM_U1TIMEOUT_MASK);
2887 xudc_writel(xudc, val, PORTPM);
2888 }
2889
2890 if (!xudc->soc->u2_enable) {
2891 val = xudc_readl(xudc, PORTPM);
2892 val &= ~(PORTPM_U2TIMEOUT_MASK);
2893 xudc_writel(xudc, val, PORTPM);
2894 }
2895
2896 if (xudc->gadget.speed <= USB_SPEED_HIGH) {
2897 val = xudc_readl(xudc, PORTPM);
2898 val &= ~(PORTPM_L1S_MASK);
2899 if (xudc->soc->lpm_enable)
2900 val |= PORTPM_L1S(PORTPM_L1S_ACCEPT);
2901 else
2902 val |= PORTPM_L1S(PORTPM_L1S_NYET);
2903 xudc_writel(xudc, val, PORTPM);
2904 }
2905
2906 val = xudc_readl(xudc, ST);
2907 if (val & ST_RC)
2908 xudc_writel(xudc, ST_RC, ST);
2909}
2910
2911static void tegra_xudc_port_disconnect(struct tegra_xudc *xudc)
2912{
2913 tegra_xudc_reset(xudc);
2914
2915 if (xudc->driver && xudc->driver->disconnect) {
2916 spin_unlock(&xudc->lock);
2917 xudc->driver->disconnect(&xudc->gadget);
2918 spin_lock(&xudc->lock);
2919 }
2920
2921 xudc->device_state = USB_STATE_NOTATTACHED;
2922 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2923
2924 complete(&xudc->disconnect_complete);
2925}
2926
2927static void tegra_xudc_port_reset(struct tegra_xudc *xudc)
2928{
2929 tegra_xudc_reset(xudc);
2930
2931 if (xudc->driver) {
2932 spin_unlock(&xudc->lock);
2933 usb_gadget_udc_reset(&xudc->gadget, xudc->driver);
2934 spin_lock(&xudc->lock);
2935 }
2936
2937 tegra_xudc_port_connect(xudc);
2938}
2939
2940static void tegra_xudc_port_suspend(struct tegra_xudc *xudc)
2941{
2942 dev_dbg(xudc->dev, "port suspend\n");
2943
2944 xudc->resume_state = xudc->device_state;
2945 xudc->device_state = USB_STATE_SUSPENDED;
2946 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2947
2948 if (xudc->driver->suspend) {
2949 spin_unlock(&xudc->lock);
2950 xudc->driver->suspend(&xudc->gadget);
2951 spin_lock(&xudc->lock);
2952 }
2953}
2954
2955static void tegra_xudc_port_resume(struct tegra_xudc *xudc)
2956{
2957 dev_dbg(xudc->dev, "port resume\n");
2958
2959 tegra_xudc_resume_device_state(xudc);
2960
2961 if (xudc->driver->resume) {
2962 spin_unlock(&xudc->lock);
2963 xudc->driver->resume(&xudc->gadget);
2964 spin_lock(&xudc->lock);
2965 }
2966}
2967
2968static inline void clear_port_change(struct tegra_xudc *xudc, u32 flag)
2969{
2970 u32 val;
2971
2972 val = xudc_readl(xudc, PORTSC);
2973 val &= ~PORTSC_CHANGE_MASK;
2974 val |= flag;
2975 xudc_writel(xudc, val, PORTSC);
2976}
2977
2978static void __tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
2979{
2980 u32 portsc, porthalt;
2981
2982 porthalt = xudc_readl(xudc, PORTHALT);
2983 if ((porthalt & PORTHALT_STCHG_REQ) &&
2984 (porthalt & PORTHALT_HALT_LTSSM)) {
2985 dev_dbg(xudc->dev, "STCHG_REQ, PORTHALT = %#x\n", porthalt);
2986 porthalt &= ~PORTHALT_HALT_LTSSM;
2987 xudc_writel(xudc, porthalt, PORTHALT);
2988 }
2989
2990 portsc = xudc_readl(xudc, PORTSC);
2991 if ((portsc & PORTSC_PRC) && (portsc & PORTSC_PR)) {
2992 dev_dbg(xudc->dev, "PRC, PR, PORTSC = %#x\n", portsc);
2993 clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
2994#define TOGGLE_VBUS_WAIT_MS 100
2995 if (xudc->soc->port_reset_quirk) {
2996 schedule_delayed_work(&xudc->port_reset_war_work,
2997 msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
2998 xudc->wait_for_sec_prc = 1;
2999 }
3000 }
3001
3002 if ((portsc & PORTSC_PRC) && !(portsc & PORTSC_PR)) {
3003 dev_dbg(xudc->dev, "PRC, Not PR, PORTSC = %#x\n", portsc);
3004 clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
3005 tegra_xudc_port_reset(xudc);
3006 cancel_delayed_work(&xudc->port_reset_war_work);
3007 xudc->wait_for_sec_prc = 0;
3008 }
3009
3010 portsc = xudc_readl(xudc, PORTSC);
3011 if (portsc & PORTSC_WRC) {
3012 dev_dbg(xudc->dev, "WRC, PORTSC = %#x\n", portsc);
3013 clear_port_change(xudc, PORTSC_WRC | PORTSC_PED);
3014 if (!(xudc_readl(xudc, PORTSC) & PORTSC_WPR))
3015 tegra_xudc_port_reset(xudc);
3016 }
3017
3018 portsc = xudc_readl(xudc, PORTSC);
3019 if (portsc & PORTSC_CSC) {
3020 dev_dbg(xudc->dev, "CSC, PORTSC = %#x\n", portsc);
3021 clear_port_change(xudc, PORTSC_CSC);
3022
3023 if (portsc & PORTSC_CCS)
3024 tegra_xudc_port_connect(xudc);
3025 else
3026 tegra_xudc_port_disconnect(xudc);
3027
3028 if (xudc->wait_csc) {
3029 cancel_delayed_work(&xudc->plc_reset_work);
3030 xudc->wait_csc = false;
3031 }
3032 }
3033
3034 portsc = xudc_readl(xudc, PORTSC);
3035 if (portsc & PORTSC_PLC) {
3036 u32 pls = (portsc & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT;
3037
3038 dev_dbg(xudc->dev, "PLC, PORTSC = %#x\n", portsc);
3039 clear_port_change(xudc, PORTSC_PLC);
3040 switch (pls) {
3041 case PORTSC_PLS_U3:
3042 tegra_xudc_port_suspend(xudc);
3043 break;
3044 case PORTSC_PLS_U0:
3045 if (xudc->gadget.speed < USB_SPEED_SUPER)
3046 tegra_xudc_port_resume(xudc);
3047 break;
3048 case PORTSC_PLS_RESUME:
3049 if (xudc->gadget.speed == USB_SPEED_SUPER)
3050 tegra_xudc_port_resume(xudc);
3051 break;
3052 case PORTSC_PLS_INACTIVE:
3053 schedule_delayed_work(&xudc->plc_reset_work,
3054 msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
3055 xudc->wait_csc = true;
3056 break;
3057 default:
3058 break;
3059 }
3060 }
3061
3062 if (portsc & PORTSC_CEC) {
3063 dev_warn(xudc->dev, "CEC, PORTSC = %#x\n", portsc);
3064 clear_port_change(xudc, PORTSC_CEC);
3065 }
3066
3067 dev_dbg(xudc->dev, "PORTSC = %#x\n", xudc_readl(xudc, PORTSC));
3068}
3069
3070static void tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
3071{
3072 while ((xudc_readl(xudc, PORTSC) & PORTSC_CHANGE_MASK) ||
3073 (xudc_readl(xudc, PORTHALT) & PORTHALT_STCHG_REQ))
3074 __tegra_xudc_handle_port_status(xudc);
3075}
3076
3077static void tegra_xudc_handle_event(struct tegra_xudc *xudc,
3078 struct tegra_xudc_trb *event)
3079{
3080 u32 type = trb_read_type(event);
3081
3082 dump_trb(xudc, "EVENT", event);
3083
3084 switch (type) {
3085 case TRB_TYPE_PORT_STATUS_CHANGE_EVENT:
3086 tegra_xudc_handle_port_status(xudc);
3087 break;
3088 case TRB_TYPE_TRANSFER_EVENT:
3089 tegra_xudc_handle_transfer_event(xudc, event);
3090 break;
3091 case TRB_TYPE_SETUP_PACKET_EVENT:
3092 tegra_xudc_handle_ep0_event(xudc, event);
3093 break;
3094 default:
3095 dev_info(xudc->dev, "Unrecognized TRB type = %#x\n", type);
3096 break;
3097 }
3098}
3099
3100static void tegra_xudc_process_event_ring(struct tegra_xudc *xudc)
3101{
3102 struct tegra_xudc_trb *event;
3103 dma_addr_t erdp;
3104
3105 while (true) {
3106 event = xudc->event_ring[xudc->event_ring_index] +
3107 xudc->event_ring_deq_ptr;
3108
3109 if (trb_read_cycle(event) != xudc->ccs)
3110 break;
3111
3112 tegra_xudc_handle_event(xudc, event);
3113
3114 xudc->event_ring_deq_ptr++;
3115 if (xudc->event_ring_deq_ptr == XUDC_EVENT_RING_SIZE) {
3116 xudc->event_ring_deq_ptr = 0;
3117 xudc->event_ring_index++;
3118 }
3119
3120 if (xudc->event_ring_index == XUDC_NR_EVENT_RINGS) {
3121 xudc->event_ring_index = 0;
3122 xudc->ccs = !xudc->ccs;
3123 }
3124 }
3125
3126 erdp = xudc->event_ring_phys[xudc->event_ring_index] +
3127 xudc->event_ring_deq_ptr * sizeof(*event);
3128
3129 xudc_writel(xudc, upper_32_bits(erdp), ERDPHI);
3130 xudc_writel(xudc, lower_32_bits(erdp) | ERDPLO_EHB, ERDPLO);
3131}
3132
3133static irqreturn_t tegra_xudc_irq(int irq, void *data)
3134{
3135 struct tegra_xudc *xudc = data;
3136 unsigned long flags;
3137 u32 val;
3138
3139 val = xudc_readl(xudc, ST);
3140 if (!(val & ST_IP))
3141 return IRQ_NONE;
3142 xudc_writel(xudc, ST_IP, ST);
3143
3144 spin_lock_irqsave(&xudc->lock, flags);
3145 tegra_xudc_process_event_ring(xudc);
3146 spin_unlock_irqrestore(&xudc->lock, flags);
3147
3148 return IRQ_HANDLED;
3149}
3150
3151static int tegra_xudc_alloc_ep(struct tegra_xudc *xudc, unsigned int index)
3152{
3153 struct tegra_xudc_ep *ep = &xudc->ep[index];
3154
3155 ep->xudc = xudc;
3156 ep->index = index;
3157 ep->context = &xudc->ep_context[index];
3158 INIT_LIST_HEAD(&ep->queue);
3159
3160 /*
3161 * EP1 would be the input endpoint corresponding to EP0, but since
3162 * EP0 is bi-directional, EP1 is unused.
3163 */
3164 if (index == 1)
3165 return 0;
3166
3167 ep->transfer_ring = dma_pool_alloc(xudc->transfer_ring_pool,
3168 GFP_KERNEL,
3169 &ep->transfer_ring_phys);
3170 if (!ep->transfer_ring)
3171 return -ENOMEM;
3172
3173 if (index) {
3174 snprintf(ep->name, sizeof(ep->name), "ep%u%s", index / 2,
3175 (index % 2 == 0) ? "out" : "in");
3176 ep->usb_ep.name = ep->name;
3177 usb_ep_set_maxpacket_limit(&ep->usb_ep, 1024);
3178 ep->usb_ep.max_streams = 16;
3179 ep->usb_ep.ops = &tegra_xudc_ep_ops;
3180 ep->usb_ep.caps.type_bulk = true;
3181 ep->usb_ep.caps.type_int = true;
3182 if (index & 1)
3183 ep->usb_ep.caps.dir_in = true;
3184 else
3185 ep->usb_ep.caps.dir_out = true;
3186 list_add_tail(&ep->usb_ep.ep_list, &xudc->gadget.ep_list);
3187 } else {
3188 strscpy(ep->name, "ep0", 3);
3189 ep->usb_ep.name = ep->name;
3190 usb_ep_set_maxpacket_limit(&ep->usb_ep, 512);
3191 ep->usb_ep.ops = &tegra_xudc_ep0_ops;
3192 ep->usb_ep.caps.type_control = true;
3193 ep->usb_ep.caps.dir_in = true;
3194 ep->usb_ep.caps.dir_out = true;
3195 }
3196
3197 return 0;
3198}
3199
3200static void tegra_xudc_free_ep(struct tegra_xudc *xudc, unsigned int index)
3201{
3202 struct tegra_xudc_ep *ep = &xudc->ep[index];
3203
3204 /*
3205 * EP1 would be the input endpoint corresponding to EP0, but since
3206 * EP0 is bi-directional, EP1 is unused.
3207 */
3208 if (index == 1)
3209 return;
3210
3211 dma_pool_free(xudc->transfer_ring_pool, ep->transfer_ring,
3212 ep->transfer_ring_phys);
3213}
3214
3215static int tegra_xudc_alloc_eps(struct tegra_xudc *xudc)
3216{
3217 struct usb_request *req;
3218 unsigned int i;
3219 int err;
3220
3221 xudc->ep_context =
3222 dma_alloc_coherent(xudc->dev, XUDC_NR_EPS *
3223 sizeof(*xudc->ep_context),
3224 &xudc->ep_context_phys, GFP_KERNEL);
3225 if (!xudc->ep_context)
3226 return -ENOMEM;
3227
3228 xudc->transfer_ring_pool =
3229 dmam_pool_create(dev_name(xudc->dev), xudc->dev,
3230 XUDC_TRANSFER_RING_SIZE *
3231 sizeof(struct tegra_xudc_trb),
3232 sizeof(struct tegra_xudc_trb), 0);
3233 if (!xudc->transfer_ring_pool) {
3234 err = -ENOMEM;
3235 goto free_ep_context;
3236 }
3237
3238 INIT_LIST_HEAD(&xudc->gadget.ep_list);
3239 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
3240 err = tegra_xudc_alloc_ep(xudc, i);
3241 if (err < 0)
3242 goto free_eps;
3243 }
3244
3245 req = tegra_xudc_ep_alloc_request(&xudc->ep[0].usb_ep, GFP_KERNEL);
3246 if (!req) {
3247 err = -ENOMEM;
3248 goto free_eps;
3249 }
3250 xudc->ep0_req = to_xudc_req(req);
3251
3252 return 0;
3253
3254free_eps:
3255 for (; i > 0; i--)
3256 tegra_xudc_free_ep(xudc, i - 1);
3257free_ep_context:
3258 dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3259 xudc->ep_context, xudc->ep_context_phys);
3260 return err;
3261}
3262
3263static void tegra_xudc_init_eps(struct tegra_xudc *xudc)
3264{
3265 xudc_writel(xudc, lower_32_bits(xudc->ep_context_phys), ECPLO);
3266 xudc_writel(xudc, upper_32_bits(xudc->ep_context_phys), ECPHI);
3267}
3268
3269static void tegra_xudc_free_eps(struct tegra_xudc *xudc)
3270{
3271 unsigned int i;
3272
3273 tegra_xudc_ep_free_request(&xudc->ep[0].usb_ep,
3274 &xudc->ep0_req->usb_req);
3275
3276 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
3277 tegra_xudc_free_ep(xudc, i);
3278
3279 dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3280 xudc->ep_context, xudc->ep_context_phys);
3281}
3282
3283static int tegra_xudc_alloc_event_ring(struct tegra_xudc *xudc)
3284{
3285 unsigned int i;
3286
3287 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3288 xudc->event_ring[i] =
3289 dma_alloc_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3290 sizeof(*xudc->event_ring[i]),
3291 &xudc->event_ring_phys[i],
3292 GFP_KERNEL);
3293 if (!xudc->event_ring[i])
3294 goto free_dma;
3295 }
3296
3297 return 0;
3298
3299free_dma:
3300 for (; i > 0; i--) {
3301 dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3302 sizeof(*xudc->event_ring[i - 1]),
3303 xudc->event_ring[i - 1],
3304 xudc->event_ring_phys[i - 1]);
3305 }
3306 return -ENOMEM;
3307}
3308
3309static void tegra_xudc_init_event_ring(struct tegra_xudc *xudc)
3310{
3311 unsigned int i;
3312 u32 val;
3313
3314 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3315 memset(xudc->event_ring[i], 0, XUDC_EVENT_RING_SIZE *
3316 sizeof(*xudc->event_ring[i]));
3317
3318 val = xudc_readl(xudc, ERSTSZ);
3319 val &= ~(ERSTSZ_ERSTXSZ_MASK << ERSTSZ_ERSTXSZ_SHIFT(i));
3320 val |= XUDC_EVENT_RING_SIZE << ERSTSZ_ERSTXSZ_SHIFT(i);
3321 xudc_writel(xudc, val, ERSTSZ);
3322
3323 xudc_writel(xudc, lower_32_bits(xudc->event_ring_phys[i]),
3324 ERSTXBALO(i));
3325 xudc_writel(xudc, upper_32_bits(xudc->event_ring_phys[i]),
3326 ERSTXBAHI(i));
3327 }
3328
3329 val = lower_32_bits(xudc->event_ring_phys[0]);
3330 xudc_writel(xudc, val, ERDPLO);
3331 val |= EREPLO_ECS;
3332 xudc_writel(xudc, val, EREPLO);
3333
3334 val = upper_32_bits(xudc->event_ring_phys[0]);
3335 xudc_writel(xudc, val, ERDPHI);
3336 xudc_writel(xudc, val, EREPHI);
3337
3338 xudc->ccs = true;
3339 xudc->event_ring_index = 0;
3340 xudc->event_ring_deq_ptr = 0;
3341}
3342
3343static void tegra_xudc_free_event_ring(struct tegra_xudc *xudc)
3344{
3345 unsigned int i;
3346
3347 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3348 dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3349 sizeof(*xudc->event_ring[i]),
3350 xudc->event_ring[i],
3351 xudc->event_ring_phys[i]);
3352 }
3353}
3354
3355static void tegra_xudc_fpci_ipfs_init(struct tegra_xudc *xudc)
3356{
3357 u32 val;
3358
3359 if (xudc->soc->has_ipfs) {
3360 val = ipfs_readl(xudc, XUSB_DEV_CONFIGURATION_0);
3361 val |= XUSB_DEV_CONFIGURATION_0_EN_FPCI;
3362 ipfs_writel(xudc, val, XUSB_DEV_CONFIGURATION_0);
3363 usleep_range(10, 15);
3364 }
3365
3366 /* Enable bus master */
3367 val = XUSB_DEV_CFG_1_IO_SPACE_EN | XUSB_DEV_CFG_1_MEMORY_SPACE_EN |
3368 XUSB_DEV_CFG_1_BUS_MASTER_EN;
3369 fpci_writel(xudc, val, XUSB_DEV_CFG_1);
3370
3371 /* Program BAR0 space */
3372 val = fpci_readl(xudc, XUSB_DEV_CFG_4);
3373 val &= ~(XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3374 val |= xudc->phys_base & (XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3375
3376 fpci_writel(xudc, val, XUSB_DEV_CFG_4);
3377 fpci_writel(xudc, upper_32_bits(xudc->phys_base), XUSB_DEV_CFG_5);
3378
3379 usleep_range(100, 200);
3380
3381 if (xudc->soc->has_ipfs) {
3382 /* Enable interrupt assertion */
3383 val = ipfs_readl(xudc, XUSB_DEV_INTR_MASK_0);
3384 val |= XUSB_DEV_INTR_MASK_0_IP_INT_MASK;
3385 ipfs_writel(xudc, val, XUSB_DEV_INTR_MASK_0);
3386 }
3387}
3388
3389static void tegra_xudc_device_params_init(struct tegra_xudc *xudc)
3390{
3391 u32 val, imod;
3392
3393 if (xudc->soc->has_ipfs) {
3394 val = xudc_readl(xudc, BLCG);
3395 val |= BLCG_ALL;
3396 val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE |
3397 BLCG_COREPLL_PWRDN);
3398 val |= BLCG_IOPLL_0_PWRDN;
3399 val |= BLCG_IOPLL_1_PWRDN;
3400 val |= BLCG_IOPLL_2_PWRDN;
3401
3402 xudc_writel(xudc, val, BLCG);
3403 }
3404
3405 if (xudc->soc->port_speed_quirk)
3406 tegra_xudc_limit_port_speed(xudc);
3407
3408 /* Set a reasonable U3 exit timer value. */
3409 val = xudc_readl(xudc, SSPX_CORE_PADCTL4);
3410 val &= ~(SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK);
3411 val |= SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(0x5dc0);
3412 xudc_writel(xudc, val, SSPX_CORE_PADCTL4);
3413
3414 /* Default ping LFPS tBurst is too large. */
3415 val = xudc_readl(xudc, SSPX_CORE_CNT0);
3416 val &= ~(SSPX_CORE_CNT0_PING_TBURST_MASK);
3417 val |= SSPX_CORE_CNT0_PING_TBURST(0xa);
3418 xudc_writel(xudc, val, SSPX_CORE_CNT0);
3419
3420 /* Default tPortConfiguration timeout is too small. */
3421 val = xudc_readl(xudc, SSPX_CORE_CNT30);
3422 val &= ~(SSPX_CORE_CNT30_LMPITP_TIMER_MASK);
3423 val |= SSPX_CORE_CNT30_LMPITP_TIMER(0x978);
3424 xudc_writel(xudc, val, SSPX_CORE_CNT30);
3425
3426 if (xudc->soc->lpm_enable) {
3427 /* Set L1 resume duration to 95 us. */
3428 val = xudc_readl(xudc, HSFSPI_COUNT13);
3429 val &= ~(HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK);
3430 val |= HSFSPI_COUNT13_U2_RESUME_K_DURATION(0x2c88);
3431 xudc_writel(xudc, val, HSFSPI_COUNT13);
3432 }
3433
3434 /*
3435 * Compliance suite appears to be violating polling LFPS tBurst max
3436 * of 1.4us. Send 1.45us instead.
3437 */
3438 val = xudc_readl(xudc, SSPX_CORE_CNT32);
3439 val &= ~(SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK);
3440 val |= SSPX_CORE_CNT32_POLL_TBURST_MAX(0xb0);
3441 xudc_writel(xudc, val, SSPX_CORE_CNT32);
3442
3443 /* Direct HS/FS port instance to RxDetect. */
3444 val = xudc_readl(xudc, CFG_DEV_FE);
3445 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3446 val |= CFG_DEV_FE_PORTREGSEL(CFG_DEV_FE_PORTREGSEL_HSFS_PI);
3447 xudc_writel(xudc, val, CFG_DEV_FE);
3448
3449 val = xudc_readl(xudc, PORTSC);
3450 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3451 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3452 xudc_writel(xudc, val, PORTSC);
3453
3454 /* Direct SS port instance to RxDetect. */
3455 val = xudc_readl(xudc, CFG_DEV_FE);
3456 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3457 val |= CFG_DEV_FE_PORTREGSEL_SS_PI & CFG_DEV_FE_PORTREGSEL_MASK;
3458 xudc_writel(xudc, val, CFG_DEV_FE);
3459
3460 val = xudc_readl(xudc, PORTSC);
3461 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3462 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3463 xudc_writel(xudc, val, PORTSC);
3464
3465 /* Restore port instance. */
3466 val = xudc_readl(xudc, CFG_DEV_FE);
3467 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3468 xudc_writel(xudc, val, CFG_DEV_FE);
3469
3470 /*
3471 * Enable INFINITE_SS_RETRY to prevent device from entering
3472 * Disabled.Error when attached to buggy SuperSpeed hubs.
3473 */
3474 val = xudc_readl(xudc, CFG_DEV_FE);
3475 val |= CFG_DEV_FE_INFINITE_SS_RETRY;
3476 xudc_writel(xudc, val, CFG_DEV_FE);
3477
3478 /* Set interrupt moderation. */
3479 imod = XUDC_INTERRUPT_MODERATION_US * 4;
3480 val = xudc_readl(xudc, RT_IMOD);
3481 val &= ~((RT_IMOD_IMODI_MASK) | (RT_IMOD_IMODC_MASK));
3482 val |= (RT_IMOD_IMODI(imod) | RT_IMOD_IMODC(imod));
3483 xudc_writel(xudc, val, RT_IMOD);
3484
3485 /* increase SSPI transaction timeout from 32us to 512us */
3486 val = xudc_readl(xudc, CFG_DEV_SSPI_XFER);
3487 val &= ~(CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK);
3488 val |= CFG_DEV_SSPI_XFER_ACKTIMEOUT(0xf000);
3489 xudc_writel(xudc, val, CFG_DEV_SSPI_XFER);
3490}
3491
3492static int tegra_xudc_phy_get(struct tegra_xudc *xudc)
3493{
3494 int err = 0, usb3;
3495 unsigned int i;
3496
3497 xudc->utmi_phy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3498 sizeof(*xudc->utmi_phy), GFP_KERNEL);
3499 if (!xudc->utmi_phy)
3500 return -ENOMEM;
3501
3502 xudc->usb3_phy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3503 sizeof(*xudc->usb3_phy), GFP_KERNEL);
3504 if (!xudc->usb3_phy)
3505 return -ENOMEM;
3506
3507 xudc->usbphy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3508 sizeof(*xudc->usbphy), GFP_KERNEL);
3509 if (!xudc->usbphy)
3510 return -ENOMEM;
3511
3512 xudc->vbus_nb.notifier_call = tegra_xudc_vbus_notify;
3513
3514 for (i = 0; i < xudc->soc->num_phys; i++) {
3515 char phy_name[] = "usb.-.";
3516
3517 /* Get USB2 phy */
3518 snprintf(phy_name, sizeof(phy_name), "usb2-%d", i);
3519 xudc->utmi_phy[i] = devm_phy_optional_get(xudc->dev, phy_name);
3520 if (IS_ERR(xudc->utmi_phy[i])) {
3521 err = PTR_ERR(xudc->utmi_phy[i]);
3522 dev_err_probe(xudc->dev, err,
3523 "failed to get usb2-%d PHY\n", i);
3524 goto clean_up;
3525 } else if (xudc->utmi_phy[i]) {
3526 /* Get usb-phy, if utmi phy is available */
3527 xudc->usbphy[i] = devm_usb_get_phy_by_node(xudc->dev,
3528 xudc->utmi_phy[i]->dev.of_node,
3529 NULL);
3530 if (IS_ERR(xudc->usbphy[i])) {
3531 err = PTR_ERR(xudc->usbphy[i]);
3532 dev_err_probe(xudc->dev, err,
3533 "failed to get usbphy-%d\n", i);
3534 goto clean_up;
3535 }
3536 } else if (!xudc->utmi_phy[i]) {
3537 /* if utmi phy is not available, ignore USB3 phy get */
3538 continue;
3539 }
3540
3541 /* Get USB3 phy */
3542 usb3 = tegra_xusb_padctl_get_usb3_companion(xudc->padctl, i);
3543 if (usb3 < 0)
3544 continue;
3545
3546 snprintf(phy_name, sizeof(phy_name), "usb3-%d", usb3);
3547 xudc->usb3_phy[i] = devm_phy_optional_get(xudc->dev, phy_name);
3548 if (IS_ERR(xudc->usb3_phy[i])) {
3549 err = PTR_ERR(xudc->usb3_phy[i]);
3550 dev_err_probe(xudc->dev, err,
3551 "failed to get usb3-%d PHY\n", usb3);
3552 goto clean_up;
3553 } else if (xudc->usb3_phy[i])
3554 dev_dbg(xudc->dev, "usb3-%d PHY registered", usb3);
3555 }
3556
3557 return err;
3558
3559clean_up:
3560 for (i = 0; i < xudc->soc->num_phys; i++) {
3561 xudc->usb3_phy[i] = NULL;
3562 xudc->utmi_phy[i] = NULL;
3563 xudc->usbphy[i] = NULL;
3564 }
3565
3566 return err;
3567}
3568
3569static void tegra_xudc_phy_exit(struct tegra_xudc *xudc)
3570{
3571 unsigned int i;
3572
3573 for (i = 0; i < xudc->soc->num_phys; i++) {
3574 phy_exit(xudc->usb3_phy[i]);
3575 phy_exit(xudc->utmi_phy[i]);
3576 }
3577}
3578
3579static int tegra_xudc_phy_init(struct tegra_xudc *xudc)
3580{
3581 int err;
3582 unsigned int i;
3583
3584 for (i = 0; i < xudc->soc->num_phys; i++) {
3585 err = phy_init(xudc->utmi_phy[i]);
3586 if (err < 0) {
3587 dev_err(xudc->dev, "UTMI PHY #%u initialization failed: %d\n", i, err);
3588 goto exit_phy;
3589 }
3590
3591 err = phy_init(xudc->usb3_phy[i]);
3592 if (err < 0) {
3593 dev_err(xudc->dev, "USB3 PHY #%u initialization failed: %d\n", i, err);
3594 goto exit_phy;
3595 }
3596 }
3597 return 0;
3598
3599exit_phy:
3600 tegra_xudc_phy_exit(xudc);
3601 return err;
3602}
3603
3604static const char * const tegra210_xudc_supply_names[] = {
3605 "hvdd-usb",
3606 "avddio-usb",
3607};
3608
3609static const char * const tegra210_xudc_clock_names[] = {
3610 "dev",
3611 "ss",
3612 "ss_src",
3613 "hs_src",
3614 "fs_src",
3615};
3616
3617static const char * const tegra186_xudc_clock_names[] = {
3618 "dev",
3619 "ss",
3620 "ss_src",
3621 "fs_src",
3622};
3623
3624static struct tegra_xudc_soc tegra210_xudc_soc_data = {
3625 .supply_names = tegra210_xudc_supply_names,
3626 .num_supplies = ARRAY_SIZE(tegra210_xudc_supply_names),
3627 .clock_names = tegra210_xudc_clock_names,
3628 .num_clks = ARRAY_SIZE(tegra210_xudc_clock_names),
3629 .num_phys = 4,
3630 .u1_enable = false,
3631 .u2_enable = true,
3632 .lpm_enable = false,
3633 .invalid_seq_num = true,
3634 .pls_quirk = true,
3635 .port_reset_quirk = true,
3636 .port_speed_quirk = false,
3637 .has_ipfs = true,
3638};
3639
3640static struct tegra_xudc_soc tegra186_xudc_soc_data = {
3641 .clock_names = tegra186_xudc_clock_names,
3642 .num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3643 .num_phys = 4,
3644 .u1_enable = true,
3645 .u2_enable = true,
3646 .lpm_enable = false,
3647 .invalid_seq_num = false,
3648 .pls_quirk = false,
3649 .port_reset_quirk = false,
3650 .port_speed_quirk = false,
3651 .has_ipfs = false,
3652};
3653
3654static struct tegra_xudc_soc tegra194_xudc_soc_data = {
3655 .clock_names = tegra186_xudc_clock_names,
3656 .num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3657 .num_phys = 4,
3658 .u1_enable = true,
3659 .u2_enable = true,
3660 .lpm_enable = true,
3661 .invalid_seq_num = false,
3662 .pls_quirk = false,
3663 .port_reset_quirk = false,
3664 .port_speed_quirk = true,
3665 .has_ipfs = false,
3666};
3667
3668static struct tegra_xudc_soc tegra234_xudc_soc_data = {
3669 .clock_names = tegra186_xudc_clock_names,
3670 .num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3671 .num_phys = 4,
3672 .u1_enable = true,
3673 .u2_enable = true,
3674 .lpm_enable = true,
3675 .invalid_seq_num = false,
3676 .pls_quirk = false,
3677 .port_reset_quirk = false,
3678 .has_ipfs = false,
3679};
3680
3681static const struct of_device_id tegra_xudc_of_match[] = {
3682 {
3683 .compatible = "nvidia,tegra210-xudc",
3684 .data = &tegra210_xudc_soc_data
3685 },
3686 {
3687 .compatible = "nvidia,tegra186-xudc",
3688 .data = &tegra186_xudc_soc_data
3689 },
3690 {
3691 .compatible = "nvidia,tegra194-xudc",
3692 .data = &tegra194_xudc_soc_data
3693 },
3694 {
3695 .compatible = "nvidia,tegra234-xudc",
3696 .data = &tegra234_xudc_soc_data
3697 },
3698 { }
3699};
3700MODULE_DEVICE_TABLE(of, tegra_xudc_of_match);
3701
3702static void tegra_xudc_powerdomain_remove(struct tegra_xudc *xudc)
3703{
3704 if (xudc->genpd_dl_ss)
3705 device_link_del(xudc->genpd_dl_ss);
3706 if (xudc->genpd_dl_device)
3707 device_link_del(xudc->genpd_dl_device);
3708 if (xudc->genpd_dev_ss)
3709 dev_pm_domain_detach(xudc->genpd_dev_ss, true);
3710 if (xudc->genpd_dev_device)
3711 dev_pm_domain_detach(xudc->genpd_dev_device, true);
3712}
3713
3714static int tegra_xudc_powerdomain_init(struct tegra_xudc *xudc)
3715{
3716 struct device *dev = xudc->dev;
3717 int err;
3718
3719 xudc->genpd_dev_device = dev_pm_domain_attach_by_name(dev, "dev");
3720 if (IS_ERR(xudc->genpd_dev_device)) {
3721 err = PTR_ERR(xudc->genpd_dev_device);
3722 dev_err(dev, "failed to get device power domain: %d\n", err);
3723 return err;
3724 }
3725
3726 xudc->genpd_dev_ss = dev_pm_domain_attach_by_name(dev, "ss");
3727 if (IS_ERR(xudc->genpd_dev_ss)) {
3728 err = PTR_ERR(xudc->genpd_dev_ss);
3729 dev_err(dev, "failed to get SuperSpeed power domain: %d\n", err);
3730 return err;
3731 }
3732
3733 xudc->genpd_dl_device = device_link_add(dev, xudc->genpd_dev_device,
3734 DL_FLAG_PM_RUNTIME |
3735 DL_FLAG_STATELESS);
3736 if (!xudc->genpd_dl_device) {
3737 dev_err(dev, "failed to add USB device link\n");
3738 return -ENODEV;
3739 }
3740
3741 xudc->genpd_dl_ss = device_link_add(dev, xudc->genpd_dev_ss,
3742 DL_FLAG_PM_RUNTIME |
3743 DL_FLAG_STATELESS);
3744 if (!xudc->genpd_dl_ss) {
3745 dev_err(dev, "failed to add SuperSpeed device link\n");
3746 return -ENODEV;
3747 }
3748
3749 return 0;
3750}
3751
3752static int tegra_xudc_probe(struct platform_device *pdev)
3753{
3754 struct tegra_xudc *xudc;
3755 struct resource *res;
3756 unsigned int i;
3757 int err;
3758
3759 xudc = devm_kzalloc(&pdev->dev, sizeof(*xudc), GFP_KERNEL);
3760 if (!xudc)
3761 return -ENOMEM;
3762
3763 xudc->dev = &pdev->dev;
3764 platform_set_drvdata(pdev, xudc);
3765
3766 xudc->soc = of_device_get_match_data(&pdev->dev);
3767 if (!xudc->soc)
3768 return -ENODEV;
3769
3770 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3771 xudc->base = devm_ioremap_resource(&pdev->dev, res);
3772 if (IS_ERR(xudc->base))
3773 return PTR_ERR(xudc->base);
3774 xudc->phys_base = res->start;
3775
3776 xudc->fpci = devm_platform_ioremap_resource_byname(pdev, "fpci");
3777 if (IS_ERR(xudc->fpci))
3778 return PTR_ERR(xudc->fpci);
3779
3780 if (xudc->soc->has_ipfs) {
3781 xudc->ipfs = devm_platform_ioremap_resource_byname(pdev, "ipfs");
3782 if (IS_ERR(xudc->ipfs))
3783 return PTR_ERR(xudc->ipfs);
3784 }
3785
3786 xudc->irq = platform_get_irq(pdev, 0);
3787 if (xudc->irq < 0)
3788 return xudc->irq;
3789
3790 err = devm_request_irq(&pdev->dev, xudc->irq, tegra_xudc_irq, 0,
3791 dev_name(&pdev->dev), xudc);
3792 if (err < 0) {
3793 dev_err(xudc->dev, "failed to claim IRQ#%u: %d\n", xudc->irq,
3794 err);
3795 return err;
3796 }
3797
3798 xudc->clks = devm_kcalloc(&pdev->dev, xudc->soc->num_clks, sizeof(*xudc->clks),
3799 GFP_KERNEL);
3800 if (!xudc->clks)
3801 return -ENOMEM;
3802
3803 for (i = 0; i < xudc->soc->num_clks; i++)
3804 xudc->clks[i].id = xudc->soc->clock_names[i];
3805
3806 err = devm_clk_bulk_get(&pdev->dev, xudc->soc->num_clks, xudc->clks);
3807 if (err) {
3808 dev_err_probe(xudc->dev, err, "failed to request clocks\n");
3809 return err;
3810 }
3811
3812 xudc->supplies = devm_kcalloc(&pdev->dev, xudc->soc->num_supplies,
3813 sizeof(*xudc->supplies), GFP_KERNEL);
3814 if (!xudc->supplies)
3815 return -ENOMEM;
3816
3817 for (i = 0; i < xudc->soc->num_supplies; i++)
3818 xudc->supplies[i].supply = xudc->soc->supply_names[i];
3819
3820 err = devm_regulator_bulk_get(&pdev->dev, xudc->soc->num_supplies,
3821 xudc->supplies);
3822 if (err) {
3823 dev_err_probe(xudc->dev, err, "failed to request regulators\n");
3824 return err;
3825 }
3826
3827 xudc->padctl = tegra_xusb_padctl_get(&pdev->dev);
3828 if (IS_ERR(xudc->padctl))
3829 return PTR_ERR(xudc->padctl);
3830
3831 err = regulator_bulk_enable(xudc->soc->num_supplies, xudc->supplies);
3832 if (err) {
3833 dev_err(xudc->dev, "failed to enable regulators: %d\n", err);
3834 goto put_padctl;
3835 }
3836
3837 err = tegra_xudc_phy_get(xudc);
3838 if (err)
3839 goto disable_regulator;
3840
3841 err = tegra_xudc_powerdomain_init(xudc);
3842 if (err)
3843 goto put_powerdomains;
3844
3845 err = tegra_xudc_phy_init(xudc);
3846 if (err)
3847 goto put_powerdomains;
3848
3849 err = tegra_xudc_alloc_event_ring(xudc);
3850 if (err)
3851 goto disable_phy;
3852
3853 err = tegra_xudc_alloc_eps(xudc);
3854 if (err)
3855 goto free_event_ring;
3856
3857 spin_lock_init(&xudc->lock);
3858
3859 init_completion(&xudc->disconnect_complete);
3860
3861 INIT_WORK(&xudc->usb_role_sw_work, tegra_xudc_usb_role_sw_work);
3862
3863 INIT_DELAYED_WORK(&xudc->plc_reset_work, tegra_xudc_plc_reset_work);
3864
3865 INIT_DELAYED_WORK(&xudc->port_reset_war_work,
3866 tegra_xudc_port_reset_war_work);
3867
3868 pm_runtime_enable(&pdev->dev);
3869
3870 xudc->gadget.ops = &tegra_xudc_gadget_ops;
3871 xudc->gadget.ep0 = &xudc->ep[0].usb_ep;
3872 xudc->gadget.name = "tegra-xudc";
3873 xudc->gadget.max_speed = USB_SPEED_SUPER;
3874
3875 err = usb_add_gadget_udc(&pdev->dev, &xudc->gadget);
3876 if (err) {
3877 dev_err(&pdev->dev, "failed to add USB gadget: %d\n", err);
3878 goto free_eps;
3879 }
3880
3881 for (i = 0; i < xudc->soc->num_phys; i++) {
3882 if (!xudc->usbphy[i])
3883 continue;
3884
3885 usb_register_notifier(xudc->usbphy[i], &xudc->vbus_nb);
3886 tegra_xudc_update_data_role(xudc, xudc->usbphy[i]);
3887 }
3888
3889 return 0;
3890
3891free_eps:
3892 pm_runtime_disable(&pdev->dev);
3893 tegra_xudc_free_eps(xudc);
3894free_event_ring:
3895 tegra_xudc_free_event_ring(xudc);
3896disable_phy:
3897 tegra_xudc_phy_exit(xudc);
3898put_powerdomains:
3899 tegra_xudc_powerdomain_remove(xudc);
3900disable_regulator:
3901 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3902put_padctl:
3903 tegra_xusb_padctl_put(xudc->padctl);
3904
3905 return err;
3906}
3907
3908static void tegra_xudc_remove(struct platform_device *pdev)
3909{
3910 struct tegra_xudc *xudc = platform_get_drvdata(pdev);
3911 unsigned int i;
3912
3913 pm_runtime_get_sync(xudc->dev);
3914
3915 cancel_delayed_work_sync(&xudc->plc_reset_work);
3916 cancel_work_sync(&xudc->usb_role_sw_work);
3917
3918 usb_del_gadget_udc(&xudc->gadget);
3919
3920 tegra_xudc_free_eps(xudc);
3921 tegra_xudc_free_event_ring(xudc);
3922
3923 tegra_xudc_powerdomain_remove(xudc);
3924
3925 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3926
3927 for (i = 0; i < xudc->soc->num_phys; i++) {
3928 phy_power_off(xudc->utmi_phy[i]);
3929 phy_power_off(xudc->usb3_phy[i]);
3930 }
3931
3932 tegra_xudc_phy_exit(xudc);
3933
3934 pm_runtime_disable(xudc->dev);
3935 pm_runtime_put(xudc->dev);
3936
3937 tegra_xusb_padctl_put(xudc->padctl);
3938}
3939
3940static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc)
3941{
3942 unsigned long flags;
3943
3944 dev_dbg(xudc->dev, "entering ELPG\n");
3945
3946 spin_lock_irqsave(&xudc->lock, flags);
3947
3948 xudc->powergated = true;
3949 xudc->saved_regs.ctrl = xudc_readl(xudc, CTRL);
3950 xudc->saved_regs.portpm = xudc_readl(xudc, PORTPM);
3951 xudc_writel(xudc, 0, CTRL);
3952
3953 spin_unlock_irqrestore(&xudc->lock, flags);
3954
3955 clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks);
3956
3957 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3958
3959 dev_dbg(xudc->dev, "entering ELPG done\n");
3960 return 0;
3961}
3962
3963static int __maybe_unused tegra_xudc_unpowergate(struct tegra_xudc *xudc)
3964{
3965 unsigned long flags;
3966 int err;
3967
3968 dev_dbg(xudc->dev, "exiting ELPG\n");
3969
3970 err = regulator_bulk_enable(xudc->soc->num_supplies,
3971 xudc->supplies);
3972 if (err < 0)
3973 return err;
3974
3975 err = clk_bulk_prepare_enable(xudc->soc->num_clks, xudc->clks);
3976 if (err < 0)
3977 return err;
3978
3979 tegra_xudc_fpci_ipfs_init(xudc);
3980
3981 tegra_xudc_device_params_init(xudc);
3982
3983 tegra_xudc_init_event_ring(xudc);
3984
3985 tegra_xudc_init_eps(xudc);
3986
3987 xudc_writel(xudc, xudc->saved_regs.portpm, PORTPM);
3988 xudc_writel(xudc, xudc->saved_regs.ctrl, CTRL);
3989
3990 spin_lock_irqsave(&xudc->lock, flags);
3991 xudc->powergated = false;
3992 spin_unlock_irqrestore(&xudc->lock, flags);
3993
3994 dev_dbg(xudc->dev, "exiting ELPG done\n");
3995 return 0;
3996}
3997
3998static int __maybe_unused tegra_xudc_suspend(struct device *dev)
3999{
4000 struct tegra_xudc *xudc = dev_get_drvdata(dev);
4001 unsigned long flags;
4002
4003 spin_lock_irqsave(&xudc->lock, flags);
4004 xudc->suspended = true;
4005 spin_unlock_irqrestore(&xudc->lock, flags);
4006
4007 flush_work(&xudc->usb_role_sw_work);
4008
4009 if (!pm_runtime_status_suspended(dev)) {
4010 /* Forcibly disconnect before powergating. */
4011 tegra_xudc_device_mode_off(xudc);
4012 tegra_xudc_powergate(xudc);
4013 }
4014
4015 pm_runtime_disable(dev);
4016
4017 return 0;
4018}
4019
4020static int __maybe_unused tegra_xudc_resume(struct device *dev)
4021{
4022 struct tegra_xudc *xudc = dev_get_drvdata(dev);
4023 unsigned long flags;
4024 int err;
4025
4026 err = tegra_xudc_unpowergate(xudc);
4027 if (err < 0)
4028 return err;
4029
4030 spin_lock_irqsave(&xudc->lock, flags);
4031 xudc->suspended = false;
4032 spin_unlock_irqrestore(&xudc->lock, flags);
4033
4034 schedule_work(&xudc->usb_role_sw_work);
4035
4036 pm_runtime_enable(dev);
4037
4038 return 0;
4039}
4040
4041static int __maybe_unused tegra_xudc_runtime_suspend(struct device *dev)
4042{
4043 struct tegra_xudc *xudc = dev_get_drvdata(dev);
4044
4045 return tegra_xudc_powergate(xudc);
4046}
4047
4048static int __maybe_unused tegra_xudc_runtime_resume(struct device *dev)
4049{
4050 struct tegra_xudc *xudc = dev_get_drvdata(dev);
4051
4052 return tegra_xudc_unpowergate(xudc);
4053}
4054
4055static const struct dev_pm_ops tegra_xudc_pm_ops = {
4056 SET_SYSTEM_SLEEP_PM_OPS(tegra_xudc_suspend, tegra_xudc_resume)
4057 SET_RUNTIME_PM_OPS(tegra_xudc_runtime_suspend,
4058 tegra_xudc_runtime_resume, NULL)
4059};
4060
4061static struct platform_driver tegra_xudc_driver = {
4062 .probe = tegra_xudc_probe,
4063 .remove_new = tegra_xudc_remove,
4064 .driver = {
4065 .name = "tegra-xudc",
4066 .pm = &tegra_xudc_pm_ops,
4067 .of_match_table = tegra_xudc_of_match,
4068 },
4069};
4070module_platform_driver(tegra_xudc_driver);
4071
4072MODULE_DESCRIPTION("NVIDIA Tegra XUSB Device Controller");
4073MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
4074MODULE_AUTHOR("Hui Fu <hfu@nvidia.com>");
4075MODULE_AUTHOR("Nagarjuna Kristam <nkristam@nvidia.com>");
4076MODULE_LICENSE("GPL v2");
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * NVIDIA Tegra XUSB device mode controller
4 *
5 * Copyright (c) 2013-2022, NVIDIA CORPORATION. All rights reserved.
6 * Copyright (c) 2015, Google Inc.
7 */
8
9#include <linux/clk.h>
10#include <linux/completion.h>
11#include <linux/delay.h>
12#include <linux/dma-mapping.h>
13#include <linux/dmapool.h>
14#include <linux/interrupt.h>
15#include <linux/iopoll.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/of.h>
19#include <linux/of_device.h>
20#include <linux/phy/phy.h>
21#include <linux/phy/tegra/xusb.h>
22#include <linux/pm_domain.h>
23#include <linux/platform_device.h>
24#include <linux/pm_runtime.h>
25#include <linux/regulator/consumer.h>
26#include <linux/reset.h>
27#include <linux/usb/ch9.h>
28#include <linux/usb/gadget.h>
29#include <linux/usb/otg.h>
30#include <linux/usb/role.h>
31#include <linux/usb/phy.h>
32#include <linux/workqueue.h>
33
34/* XUSB_DEV registers */
35#define DB 0x004
36#define DB_TARGET_MASK GENMASK(15, 8)
37#define DB_TARGET(x) (((x) << 8) & DB_TARGET_MASK)
38#define DB_STREAMID_MASK GENMASK(31, 16)
39#define DB_STREAMID(x) (((x) << 16) & DB_STREAMID_MASK)
40#define ERSTSZ 0x008
41#define ERSTSZ_ERSTXSZ_SHIFT(x) ((x) * 16)
42#define ERSTSZ_ERSTXSZ_MASK GENMASK(15, 0)
43#define ERSTXBALO(x) (0x010 + 8 * (x))
44#define ERSTXBAHI(x) (0x014 + 8 * (x))
45#define ERDPLO 0x020
46#define ERDPLO_EHB BIT(3)
47#define ERDPHI 0x024
48#define EREPLO 0x028
49#define EREPLO_ECS BIT(0)
50#define EREPLO_SEGI BIT(1)
51#define EREPHI 0x02c
52#define CTRL 0x030
53#define CTRL_RUN BIT(0)
54#define CTRL_LSE BIT(1)
55#define CTRL_IE BIT(4)
56#define CTRL_SMI_EVT BIT(5)
57#define CTRL_SMI_DSE BIT(6)
58#define CTRL_EWE BIT(7)
59#define CTRL_DEVADDR_MASK GENMASK(30, 24)
60#define CTRL_DEVADDR(x) (((x) << 24) & CTRL_DEVADDR_MASK)
61#define CTRL_ENABLE BIT(31)
62#define ST 0x034
63#define ST_RC BIT(0)
64#define ST_IP BIT(4)
65#define RT_IMOD 0x038
66#define RT_IMOD_IMODI_MASK GENMASK(15, 0)
67#define RT_IMOD_IMODI(x) ((x) & RT_IMOD_IMODI_MASK)
68#define RT_IMOD_IMODC_MASK GENMASK(31, 16)
69#define RT_IMOD_IMODC(x) (((x) << 16) & RT_IMOD_IMODC_MASK)
70#define PORTSC 0x03c
71#define PORTSC_CCS BIT(0)
72#define PORTSC_PED BIT(1)
73#define PORTSC_PR BIT(4)
74#define PORTSC_PLS_SHIFT 5
75#define PORTSC_PLS_MASK GENMASK(8, 5)
76#define PORTSC_PLS_U0 0x0
77#define PORTSC_PLS_U2 0x2
78#define PORTSC_PLS_U3 0x3
79#define PORTSC_PLS_DISABLED 0x4
80#define PORTSC_PLS_RXDETECT 0x5
81#define PORTSC_PLS_INACTIVE 0x6
82#define PORTSC_PLS_RESUME 0xf
83#define PORTSC_PLS(x) (((x) << PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK)
84#define PORTSC_PS_SHIFT 10
85#define PORTSC_PS_MASK GENMASK(13, 10)
86#define PORTSC_PS_UNDEFINED 0x0
87#define PORTSC_PS_FS 0x1
88#define PORTSC_PS_LS 0x2
89#define PORTSC_PS_HS 0x3
90#define PORTSC_PS_SS 0x4
91#define PORTSC_LWS BIT(16)
92#define PORTSC_CSC BIT(17)
93#define PORTSC_WRC BIT(19)
94#define PORTSC_PRC BIT(21)
95#define PORTSC_PLC BIT(22)
96#define PORTSC_CEC BIT(23)
97#define PORTSC_WPR BIT(30)
98#define PORTSC_CHANGE_MASK (PORTSC_CSC | PORTSC_WRC | PORTSC_PRC | \
99 PORTSC_PLC | PORTSC_CEC)
100#define ECPLO 0x040
101#define ECPHI 0x044
102#define MFINDEX 0x048
103#define MFINDEX_FRAME_SHIFT 3
104#define MFINDEX_FRAME_MASK GENMASK(13, 3)
105#define PORTPM 0x04c
106#define PORTPM_L1S_MASK GENMASK(1, 0)
107#define PORTPM_L1S_DROP 0x0
108#define PORTPM_L1S_ACCEPT 0x1
109#define PORTPM_L1S_NYET 0x2
110#define PORTPM_L1S_STALL 0x3
111#define PORTPM_L1S(x) ((x) & PORTPM_L1S_MASK)
112#define PORTPM_RWE BIT(3)
113#define PORTPM_U2TIMEOUT_MASK GENMASK(15, 8)
114#define PORTPM_U1TIMEOUT_MASK GENMASK(23, 16)
115#define PORTPM_FLA BIT(24)
116#define PORTPM_VBA BIT(25)
117#define PORTPM_WOC BIT(26)
118#define PORTPM_WOD BIT(27)
119#define PORTPM_U1E BIT(28)
120#define PORTPM_U2E BIT(29)
121#define PORTPM_FRWE BIT(30)
122#define PORTPM_PNG_CYA BIT(31)
123#define EP_HALT 0x050
124#define EP_PAUSE 0x054
125#define EP_RELOAD 0x058
126#define EP_STCHG 0x05c
127#define DEVNOTIF_LO 0x064
128#define DEVNOTIF_LO_TRIG BIT(0)
129#define DEVNOTIF_LO_TYPE_MASK GENMASK(7, 4)
130#define DEVNOTIF_LO_TYPE(x) (((x) << 4) & DEVNOTIF_LO_TYPE_MASK)
131#define DEVNOTIF_LO_TYPE_FUNCTION_WAKE 0x1
132#define DEVNOTIF_HI 0x068
133#define PORTHALT 0x06c
134#define PORTHALT_HALT_LTSSM BIT(0)
135#define PORTHALT_HALT_REJECT BIT(1)
136#define PORTHALT_STCHG_REQ BIT(20)
137#define PORTHALT_STCHG_INTR_EN BIT(24)
138#define PORT_TM 0x070
139#define EP_THREAD_ACTIVE 0x074
140#define EP_STOPPED 0x078
141#define HSFSPI_COUNT0 0x100
142#define HSFSPI_COUNT13 0x134
143#define HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK GENMASK(29, 0)
144#define HSFSPI_COUNT13_U2_RESUME_K_DURATION(x) ((x) & \
145 HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK)
146#define BLCG 0x840
147#define SSPX_CORE_CNT0 0x610
148#define SSPX_CORE_CNT0_PING_TBURST_MASK GENMASK(7, 0)
149#define SSPX_CORE_CNT0_PING_TBURST(x) ((x) & SSPX_CORE_CNT0_PING_TBURST_MASK)
150#define SSPX_CORE_CNT30 0x688
151#define SSPX_CORE_CNT30_LMPITP_TIMER_MASK GENMASK(19, 0)
152#define SSPX_CORE_CNT30_LMPITP_TIMER(x) ((x) & \
153 SSPX_CORE_CNT30_LMPITP_TIMER_MASK)
154#define SSPX_CORE_CNT32 0x690
155#define SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK GENMASK(7, 0)
156#define SSPX_CORE_CNT32_POLL_TBURST_MAX(x) ((x) & \
157 SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK)
158#define SSPX_CORE_CNT56 0x6fc
159#define SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK GENMASK(19, 0)
160#define SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(x) ((x) & \
161 SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK)
162#define SSPX_CORE_CNT57 0x700
163#define SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK GENMASK(19, 0)
164#define SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(x) ((x) & \
165 SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK)
166#define SSPX_CORE_CNT65 0x720
167#define SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK GENMASK(19, 0)
168#define SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(x) ((x) & \
169 SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK)
170#define SSPX_CORE_CNT66 0x724
171#define SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK GENMASK(19, 0)
172#define SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(x) ((x) & \
173 SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK)
174#define SSPX_CORE_CNT67 0x728
175#define SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK GENMASK(19, 0)
176#define SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(x) ((x) & \
177 SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK)
178#define SSPX_CORE_CNT72 0x73c
179#define SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK GENMASK(19, 0)
180#define SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(x) ((x) & \
181 SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK)
182#define SSPX_CORE_PADCTL4 0x750
183#define SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK GENMASK(19, 0)
184#define SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(x) ((x) & \
185 SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK)
186#define BLCG_DFPCI BIT(0)
187#define BLCG_UFPCI BIT(1)
188#define BLCG_FE BIT(2)
189#define BLCG_COREPLL_PWRDN BIT(8)
190#define BLCG_IOPLL_0_PWRDN BIT(9)
191#define BLCG_IOPLL_1_PWRDN BIT(10)
192#define BLCG_IOPLL_2_PWRDN BIT(11)
193#define BLCG_ALL 0x1ff
194#define CFG_DEV_SSPI_XFER 0x858
195#define CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK GENMASK(31, 0)
196#define CFG_DEV_SSPI_XFER_ACKTIMEOUT(x) ((x) & \
197 CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK)
198#define CFG_DEV_FE 0x85c
199#define CFG_DEV_FE_PORTREGSEL_MASK GENMASK(1, 0)
200#define CFG_DEV_FE_PORTREGSEL_SS_PI 1
201#define CFG_DEV_FE_PORTREGSEL_HSFS_PI 2
202#define CFG_DEV_FE_PORTREGSEL(x) ((x) & CFG_DEV_FE_PORTREGSEL_MASK)
203#define CFG_DEV_FE_INFINITE_SS_RETRY BIT(29)
204
205/* FPCI registers */
206#define XUSB_DEV_CFG_1 0x004
207#define XUSB_DEV_CFG_1_IO_SPACE_EN BIT(0)
208#define XUSB_DEV_CFG_1_MEMORY_SPACE_EN BIT(1)
209#define XUSB_DEV_CFG_1_BUS_MASTER_EN BIT(2)
210#define XUSB_DEV_CFG_4 0x010
211#define XUSB_DEV_CFG_4_BASE_ADDR_MASK GENMASK(31, 15)
212#define XUSB_DEV_CFG_5 0x014
213
214/* IPFS registers */
215#define XUSB_DEV_CONFIGURATION_0 0x180
216#define XUSB_DEV_CONFIGURATION_0_EN_FPCI BIT(0)
217#define XUSB_DEV_INTR_MASK_0 0x188
218#define XUSB_DEV_INTR_MASK_0_IP_INT_MASK BIT(16)
219
220struct tegra_xudc_ep_context {
221 __le32 info0;
222 __le32 info1;
223 __le32 deq_lo;
224 __le32 deq_hi;
225 __le32 tx_info;
226 __le32 rsvd[11];
227};
228
229#define EP_STATE_DISABLED 0
230#define EP_STATE_RUNNING 1
231#define EP_STATE_HALTED 2
232#define EP_STATE_STOPPED 3
233#define EP_STATE_ERROR 4
234
235#define EP_TYPE_INVALID 0
236#define EP_TYPE_ISOCH_OUT 1
237#define EP_TYPE_BULK_OUT 2
238#define EP_TYPE_INTERRUPT_OUT 3
239#define EP_TYPE_CONTROL 4
240#define EP_TYPE_ISCOH_IN 5
241#define EP_TYPE_BULK_IN 6
242#define EP_TYPE_INTERRUPT_IN 7
243
244#define BUILD_EP_CONTEXT_RW(name, member, shift, mask) \
245static inline u32 ep_ctx_read_##name(struct tegra_xudc_ep_context *ctx) \
246{ \
247 return (le32_to_cpu(ctx->member) >> (shift)) & (mask); \
248} \
249static inline void \
250ep_ctx_write_##name(struct tegra_xudc_ep_context *ctx, u32 val) \
251{ \
252 u32 tmp; \
253 \
254 tmp = le32_to_cpu(ctx->member) & ~((mask) << (shift)); \
255 tmp |= (val & (mask)) << (shift); \
256 ctx->member = cpu_to_le32(tmp); \
257}
258
259BUILD_EP_CONTEXT_RW(state, info0, 0, 0x7)
260BUILD_EP_CONTEXT_RW(mult, info0, 8, 0x3)
261BUILD_EP_CONTEXT_RW(max_pstreams, info0, 10, 0x1f)
262BUILD_EP_CONTEXT_RW(lsa, info0, 15, 0x1)
263BUILD_EP_CONTEXT_RW(interval, info0, 16, 0xff)
264BUILD_EP_CONTEXT_RW(cerr, info1, 1, 0x3)
265BUILD_EP_CONTEXT_RW(type, info1, 3, 0x7)
266BUILD_EP_CONTEXT_RW(hid, info1, 7, 0x1)
267BUILD_EP_CONTEXT_RW(max_burst_size, info1, 8, 0xff)
268BUILD_EP_CONTEXT_RW(max_packet_size, info1, 16, 0xffff)
269BUILD_EP_CONTEXT_RW(dcs, deq_lo, 0, 0x1)
270BUILD_EP_CONTEXT_RW(deq_lo, deq_lo, 4, 0xfffffff)
271BUILD_EP_CONTEXT_RW(deq_hi, deq_hi, 0, 0xffffffff)
272BUILD_EP_CONTEXT_RW(avg_trb_len, tx_info, 0, 0xffff)
273BUILD_EP_CONTEXT_RW(max_esit_payload, tx_info, 16, 0xffff)
274BUILD_EP_CONTEXT_RW(edtla, rsvd[0], 0, 0xffffff)
275BUILD_EP_CONTEXT_RW(rsvd, rsvd[0], 24, 0x1)
276BUILD_EP_CONTEXT_RW(partial_td, rsvd[0], 25, 0x1)
277BUILD_EP_CONTEXT_RW(splitxstate, rsvd[0], 26, 0x1)
278BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 27, 0x1f)
279BUILD_EP_CONTEXT_RW(cerrcnt, rsvd[1], 18, 0x3)
280BUILD_EP_CONTEXT_RW(data_offset, rsvd[2], 0, 0x1ffff)
281BUILD_EP_CONTEXT_RW(numtrbs, rsvd[2], 22, 0x1f)
282BUILD_EP_CONTEXT_RW(devaddr, rsvd[6], 0, 0x7f)
283
284static inline u64 ep_ctx_read_deq_ptr(struct tegra_xudc_ep_context *ctx)
285{
286 return ((u64)ep_ctx_read_deq_hi(ctx) << 32) |
287 (ep_ctx_read_deq_lo(ctx) << 4);
288}
289
290static inline void
291ep_ctx_write_deq_ptr(struct tegra_xudc_ep_context *ctx, u64 addr)
292{
293 ep_ctx_write_deq_lo(ctx, lower_32_bits(addr) >> 4);
294 ep_ctx_write_deq_hi(ctx, upper_32_bits(addr));
295}
296
297struct tegra_xudc_trb {
298 __le32 data_lo;
299 __le32 data_hi;
300 __le32 status;
301 __le32 control;
302};
303
304#define TRB_TYPE_RSVD 0
305#define TRB_TYPE_NORMAL 1
306#define TRB_TYPE_SETUP_STAGE 2
307#define TRB_TYPE_DATA_STAGE 3
308#define TRB_TYPE_STATUS_STAGE 4
309#define TRB_TYPE_ISOCH 5
310#define TRB_TYPE_LINK 6
311#define TRB_TYPE_TRANSFER_EVENT 32
312#define TRB_TYPE_PORT_STATUS_CHANGE_EVENT 34
313#define TRB_TYPE_STREAM 48
314#define TRB_TYPE_SETUP_PACKET_EVENT 63
315
316#define TRB_CMPL_CODE_INVALID 0
317#define TRB_CMPL_CODE_SUCCESS 1
318#define TRB_CMPL_CODE_DATA_BUFFER_ERR 2
319#define TRB_CMPL_CODE_BABBLE_DETECTED_ERR 3
320#define TRB_CMPL_CODE_USB_TRANS_ERR 4
321#define TRB_CMPL_CODE_TRB_ERR 5
322#define TRB_CMPL_CODE_STALL 6
323#define TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR 10
324#define TRB_CMPL_CODE_SHORT_PACKET 13
325#define TRB_CMPL_CODE_RING_UNDERRUN 14
326#define TRB_CMPL_CODE_RING_OVERRUN 15
327#define TRB_CMPL_CODE_EVENT_RING_FULL_ERR 21
328#define TRB_CMPL_CODE_STOPPED 26
329#define TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN 31
330#define TRB_CMPL_CODE_STREAM_NUMP_ERROR 219
331#define TRB_CMPL_CODE_PRIME_PIPE_RECEIVED 220
332#define TRB_CMPL_CODE_HOST_REJECTED 221
333#define TRB_CMPL_CODE_CTRL_DIR_ERR 222
334#define TRB_CMPL_CODE_CTRL_SEQNUM_ERR 223
335
336#define BUILD_TRB_RW(name, member, shift, mask) \
337static inline u32 trb_read_##name(struct tegra_xudc_trb *trb) \
338{ \
339 return (le32_to_cpu(trb->member) >> (shift)) & (mask); \
340} \
341static inline void \
342trb_write_##name(struct tegra_xudc_trb *trb, u32 val) \
343{ \
344 u32 tmp; \
345 \
346 tmp = le32_to_cpu(trb->member) & ~((mask) << (shift)); \
347 tmp |= (val & (mask)) << (shift); \
348 trb->member = cpu_to_le32(tmp); \
349}
350
351BUILD_TRB_RW(data_lo, data_lo, 0, 0xffffffff)
352BUILD_TRB_RW(data_hi, data_hi, 0, 0xffffffff)
353BUILD_TRB_RW(seq_num, status, 0, 0xffff)
354BUILD_TRB_RW(transfer_len, status, 0, 0xffffff)
355BUILD_TRB_RW(td_size, status, 17, 0x1f)
356BUILD_TRB_RW(cmpl_code, status, 24, 0xff)
357BUILD_TRB_RW(cycle, control, 0, 0x1)
358BUILD_TRB_RW(toggle_cycle, control, 1, 0x1)
359BUILD_TRB_RW(isp, control, 2, 0x1)
360BUILD_TRB_RW(chain, control, 4, 0x1)
361BUILD_TRB_RW(ioc, control, 5, 0x1)
362BUILD_TRB_RW(type, control, 10, 0x3f)
363BUILD_TRB_RW(stream_id, control, 16, 0xffff)
364BUILD_TRB_RW(endpoint_id, control, 16, 0x1f)
365BUILD_TRB_RW(tlbpc, control, 16, 0xf)
366BUILD_TRB_RW(data_stage_dir, control, 16, 0x1)
367BUILD_TRB_RW(frame_id, control, 20, 0x7ff)
368BUILD_TRB_RW(sia, control, 31, 0x1)
369
370static inline u64 trb_read_data_ptr(struct tegra_xudc_trb *trb)
371{
372 return ((u64)trb_read_data_hi(trb) << 32) |
373 trb_read_data_lo(trb);
374}
375
376static inline void trb_write_data_ptr(struct tegra_xudc_trb *trb, u64 addr)
377{
378 trb_write_data_lo(trb, lower_32_bits(addr));
379 trb_write_data_hi(trb, upper_32_bits(addr));
380}
381
382struct tegra_xudc_request {
383 struct usb_request usb_req;
384
385 size_t buf_queued;
386 unsigned int trbs_queued;
387 unsigned int trbs_needed;
388 bool need_zlp;
389
390 struct tegra_xudc_trb *first_trb;
391 struct tegra_xudc_trb *last_trb;
392
393 struct list_head list;
394};
395
396struct tegra_xudc_ep {
397 struct tegra_xudc *xudc;
398 struct usb_ep usb_ep;
399 unsigned int index;
400 char name[8];
401
402 struct tegra_xudc_ep_context *context;
403
404#define XUDC_TRANSFER_RING_SIZE 64
405 struct tegra_xudc_trb *transfer_ring;
406 dma_addr_t transfer_ring_phys;
407
408 unsigned int enq_ptr;
409 unsigned int deq_ptr;
410 bool pcs;
411 bool ring_full;
412 bool stream_rejected;
413
414 struct list_head queue;
415 const struct usb_endpoint_descriptor *desc;
416 const struct usb_ss_ep_comp_descriptor *comp_desc;
417};
418
419struct tegra_xudc_sel_timing {
420 __u8 u1sel;
421 __u8 u1pel;
422 __le16 u2sel;
423 __le16 u2pel;
424};
425
426enum tegra_xudc_setup_state {
427 WAIT_FOR_SETUP,
428 DATA_STAGE_XFER,
429 DATA_STAGE_RECV,
430 STATUS_STAGE_XFER,
431 STATUS_STAGE_RECV,
432};
433
434struct tegra_xudc_setup_packet {
435 struct usb_ctrlrequest ctrl_req;
436 unsigned int seq_num;
437};
438
439struct tegra_xudc_save_regs {
440 u32 ctrl;
441 u32 portpm;
442};
443
444struct tegra_xudc {
445 struct device *dev;
446 const struct tegra_xudc_soc *soc;
447 struct tegra_xusb_padctl *padctl;
448
449 spinlock_t lock;
450
451 struct usb_gadget gadget;
452 struct usb_gadget_driver *driver;
453
454#define XUDC_NR_EVENT_RINGS 2
455#define XUDC_EVENT_RING_SIZE 4096
456 struct tegra_xudc_trb *event_ring[XUDC_NR_EVENT_RINGS];
457 dma_addr_t event_ring_phys[XUDC_NR_EVENT_RINGS];
458 unsigned int event_ring_index;
459 unsigned int event_ring_deq_ptr;
460 bool ccs;
461
462#define XUDC_NR_EPS 32
463 struct tegra_xudc_ep ep[XUDC_NR_EPS];
464 struct tegra_xudc_ep_context *ep_context;
465 dma_addr_t ep_context_phys;
466
467 struct device *genpd_dev_device;
468 struct device *genpd_dev_ss;
469 struct device_link *genpd_dl_device;
470 struct device_link *genpd_dl_ss;
471
472 struct dma_pool *transfer_ring_pool;
473
474 bool queued_setup_packet;
475 struct tegra_xudc_setup_packet setup_packet;
476 enum tegra_xudc_setup_state setup_state;
477 u16 setup_seq_num;
478
479 u16 dev_addr;
480 u16 isoch_delay;
481 struct tegra_xudc_sel_timing sel_timing;
482 u8 test_mode_pattern;
483 u16 status_buf;
484 struct tegra_xudc_request *ep0_req;
485
486 bool pullup;
487
488 unsigned int nr_enabled_eps;
489 unsigned int nr_isoch_eps;
490
491 unsigned int device_state;
492 unsigned int resume_state;
493
494 int irq;
495
496 void __iomem *base;
497 resource_size_t phys_base;
498 void __iomem *ipfs;
499 void __iomem *fpci;
500
501 struct regulator_bulk_data *supplies;
502
503 struct clk_bulk_data *clks;
504
505 bool device_mode;
506 struct work_struct usb_role_sw_work;
507
508 struct phy **usb3_phy;
509 struct phy *curr_usb3_phy;
510 struct phy **utmi_phy;
511 struct phy *curr_utmi_phy;
512
513 struct tegra_xudc_save_regs saved_regs;
514 bool suspended;
515 bool powergated;
516
517 struct usb_phy **usbphy;
518 struct usb_phy *curr_usbphy;
519 struct notifier_block vbus_nb;
520
521 struct completion disconnect_complete;
522
523 bool selfpowered;
524
525#define TOGGLE_VBUS_WAIT_MS 100
526 struct delayed_work plc_reset_work;
527 bool wait_csc;
528
529 struct delayed_work port_reset_war_work;
530 bool wait_for_sec_prc;
531};
532
533#define XUDC_TRB_MAX_BUFFER_SIZE 65536
534#define XUDC_MAX_ISOCH_EPS 4
535#define XUDC_INTERRUPT_MODERATION_US 0
536
537static struct usb_endpoint_descriptor tegra_xudc_ep0_desc = {
538 .bLength = USB_DT_ENDPOINT_SIZE,
539 .bDescriptorType = USB_DT_ENDPOINT,
540 .bEndpointAddress = 0,
541 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
542 .wMaxPacketSize = cpu_to_le16(64),
543};
544
545struct tegra_xudc_soc {
546 const char * const *supply_names;
547 unsigned int num_supplies;
548 const char * const *clock_names;
549 unsigned int num_clks;
550 unsigned int num_phys;
551 bool u1_enable;
552 bool u2_enable;
553 bool lpm_enable;
554 bool invalid_seq_num;
555 bool pls_quirk;
556 bool port_reset_quirk;
557 bool port_speed_quirk;
558 bool has_ipfs;
559};
560
561static inline u32 fpci_readl(struct tegra_xudc *xudc, unsigned int offset)
562{
563 return readl(xudc->fpci + offset);
564}
565
566static inline void fpci_writel(struct tegra_xudc *xudc, u32 val,
567 unsigned int offset)
568{
569 writel(val, xudc->fpci + offset);
570}
571
572static inline u32 ipfs_readl(struct tegra_xudc *xudc, unsigned int offset)
573{
574 return readl(xudc->ipfs + offset);
575}
576
577static inline void ipfs_writel(struct tegra_xudc *xudc, u32 val,
578 unsigned int offset)
579{
580 writel(val, xudc->ipfs + offset);
581}
582
583static inline u32 xudc_readl(struct tegra_xudc *xudc, unsigned int offset)
584{
585 return readl(xudc->base + offset);
586}
587
588static inline void xudc_writel(struct tegra_xudc *xudc, u32 val,
589 unsigned int offset)
590{
591 writel(val, xudc->base + offset);
592}
593
594static inline int xudc_readl_poll(struct tegra_xudc *xudc,
595 unsigned int offset, u32 mask, u32 val)
596{
597 u32 regval;
598
599 return readl_poll_timeout_atomic(xudc->base + offset, regval,
600 (regval & mask) == val, 1, 100);
601}
602
603static inline struct tegra_xudc *to_xudc(struct usb_gadget *gadget)
604{
605 return container_of(gadget, struct tegra_xudc, gadget);
606}
607
608static inline struct tegra_xudc_ep *to_xudc_ep(struct usb_ep *ep)
609{
610 return container_of(ep, struct tegra_xudc_ep, usb_ep);
611}
612
613static inline struct tegra_xudc_request *to_xudc_req(struct usb_request *req)
614{
615 return container_of(req, struct tegra_xudc_request, usb_req);
616}
617
618static inline void dump_trb(struct tegra_xudc *xudc, const char *type,
619 struct tegra_xudc_trb *trb)
620{
621 dev_dbg(xudc->dev,
622 "%s: %p, lo = %#x, hi = %#x, status = %#x, control = %#x\n",
623 type, trb, trb->data_lo, trb->data_hi, trb->status,
624 trb->control);
625}
626
627static void tegra_xudc_limit_port_speed(struct tegra_xudc *xudc)
628{
629 u32 val;
630
631 /* limit port speed to gen 1 */
632 val = xudc_readl(xudc, SSPX_CORE_CNT56);
633 val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
634 val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x260);
635 xudc_writel(xudc, val, SSPX_CORE_CNT56);
636
637 val = xudc_readl(xudc, SSPX_CORE_CNT57);
638 val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
639 val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x6D6);
640 xudc_writel(xudc, val, SSPX_CORE_CNT57);
641
642 val = xudc_readl(xudc, SSPX_CORE_CNT65);
643 val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
644 val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0x4B0);
645 xudc_writel(xudc, val, SSPX_CORE_CNT66);
646
647 val = xudc_readl(xudc, SSPX_CORE_CNT66);
648 val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
649 val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x4B0);
650 xudc_writel(xudc, val, SSPX_CORE_CNT66);
651
652 val = xudc_readl(xudc, SSPX_CORE_CNT67);
653 val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
654 val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x4B0);
655 xudc_writel(xudc, val, SSPX_CORE_CNT67);
656
657 val = xudc_readl(xudc, SSPX_CORE_CNT72);
658 val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
659 val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x10);
660 xudc_writel(xudc, val, SSPX_CORE_CNT72);
661}
662
663static void tegra_xudc_restore_port_speed(struct tegra_xudc *xudc)
664{
665 u32 val;
666
667 /* restore port speed to gen2 */
668 val = xudc_readl(xudc, SSPX_CORE_CNT56);
669 val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
670 val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x438);
671 xudc_writel(xudc, val, SSPX_CORE_CNT56);
672
673 val = xudc_readl(xudc, SSPX_CORE_CNT57);
674 val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
675 val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x528);
676 xudc_writel(xudc, val, SSPX_CORE_CNT57);
677
678 val = xudc_readl(xudc, SSPX_CORE_CNT65);
679 val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
680 val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0xE10);
681 xudc_writel(xudc, val, SSPX_CORE_CNT66);
682
683 val = xudc_readl(xudc, SSPX_CORE_CNT66);
684 val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
685 val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x348);
686 xudc_writel(xudc, val, SSPX_CORE_CNT66);
687
688 val = xudc_readl(xudc, SSPX_CORE_CNT67);
689 val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
690 val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x5a0);
691 xudc_writel(xudc, val, SSPX_CORE_CNT67);
692
693 val = xudc_readl(xudc, SSPX_CORE_CNT72);
694 val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
695 val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x1c21);
696 xudc_writel(xudc, val, SSPX_CORE_CNT72);
697}
698
699static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc)
700{
701 int err;
702
703 pm_runtime_get_sync(xudc->dev);
704
705 tegra_phy_xusb_utmi_pad_power_on(xudc->curr_utmi_phy);
706
707 err = phy_power_on(xudc->curr_utmi_phy);
708 if (err < 0)
709 dev_err(xudc->dev, "UTMI power on failed: %d\n", err);
710
711 err = phy_power_on(xudc->curr_usb3_phy);
712 if (err < 0)
713 dev_err(xudc->dev, "USB3 PHY power on failed: %d\n", err);
714
715 dev_dbg(xudc->dev, "device mode on\n");
716
717 phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
718 USB_ROLE_DEVICE);
719}
720
721static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc)
722{
723 bool connected = false;
724 u32 pls, val;
725 int err;
726
727 dev_dbg(xudc->dev, "device mode off\n");
728
729 connected = !!(xudc_readl(xudc, PORTSC) & PORTSC_CCS);
730
731 reinit_completion(&xudc->disconnect_complete);
732
733 if (xudc->soc->port_speed_quirk)
734 tegra_xudc_restore_port_speed(xudc);
735
736 phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG, USB_ROLE_NONE);
737
738 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
739 PORTSC_PLS_SHIFT;
740
741 /* Direct link to U0 if disconnected in RESUME or U2. */
742 if (xudc->soc->pls_quirk && xudc->gadget.speed == USB_SPEED_SUPER &&
743 (pls == PORTSC_PLS_RESUME || pls == PORTSC_PLS_U2)) {
744 val = xudc_readl(xudc, PORTPM);
745 val |= PORTPM_FRWE;
746 xudc_writel(xudc, val, PORTPM);
747
748 val = xudc_readl(xudc, PORTSC);
749 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
750 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
751 xudc_writel(xudc, val, PORTSC);
752 }
753
754 /* Wait for disconnect event. */
755 if (connected)
756 wait_for_completion(&xudc->disconnect_complete);
757
758 /* Make sure interrupt handler has completed before powergating. */
759 synchronize_irq(xudc->irq);
760
761 tegra_phy_xusb_utmi_pad_power_down(xudc->curr_utmi_phy);
762
763 err = phy_power_off(xudc->curr_utmi_phy);
764 if (err < 0)
765 dev_err(xudc->dev, "UTMI PHY power off failed: %d\n", err);
766
767 err = phy_power_off(xudc->curr_usb3_phy);
768 if (err < 0)
769 dev_err(xudc->dev, "USB3 PHY power off failed: %d\n", err);
770
771 pm_runtime_put(xudc->dev);
772}
773
774static void tegra_xudc_usb_role_sw_work(struct work_struct *work)
775{
776 struct tegra_xudc *xudc = container_of(work, struct tegra_xudc,
777 usb_role_sw_work);
778
779 if (xudc->device_mode)
780 tegra_xudc_device_mode_on(xudc);
781 else
782 tegra_xudc_device_mode_off(xudc);
783}
784
785static int tegra_xudc_get_phy_index(struct tegra_xudc *xudc,
786 struct usb_phy *usbphy)
787{
788 unsigned int i;
789
790 for (i = 0; i < xudc->soc->num_phys; i++) {
791 if (xudc->usbphy[i] && usbphy == xudc->usbphy[i])
792 return i;
793 }
794
795 dev_info(xudc->dev, "phy index could not be found for shared USB PHY");
796 return -1;
797}
798
799static int tegra_xudc_vbus_notify(struct notifier_block *nb,
800 unsigned long action, void *data)
801{
802 struct tegra_xudc *xudc = container_of(nb, struct tegra_xudc,
803 vbus_nb);
804 struct usb_phy *usbphy = (struct usb_phy *)data;
805 int phy_index;
806
807 dev_dbg(xudc->dev, "%s(): event is %d\n", __func__, usbphy->last_event);
808
809 if ((xudc->device_mode && usbphy->last_event == USB_EVENT_VBUS) ||
810 (!xudc->device_mode && usbphy->last_event != USB_EVENT_VBUS)) {
811 dev_dbg(xudc->dev, "Same role(%d) received. Ignore",
812 xudc->device_mode);
813 return NOTIFY_OK;
814 }
815
816 xudc->device_mode = (usbphy->last_event == USB_EVENT_VBUS) ? true :
817 false;
818
819 phy_index = tegra_xudc_get_phy_index(xudc, usbphy);
820 dev_dbg(xudc->dev, "%s(): current phy index is %d\n", __func__,
821 phy_index);
822
823 if (!xudc->suspended && phy_index != -1) {
824 xudc->curr_utmi_phy = xudc->utmi_phy[phy_index];
825 xudc->curr_usb3_phy = xudc->usb3_phy[phy_index];
826 xudc->curr_usbphy = usbphy;
827 schedule_work(&xudc->usb_role_sw_work);
828 }
829
830 return NOTIFY_OK;
831}
832
833static void tegra_xudc_plc_reset_work(struct work_struct *work)
834{
835 struct delayed_work *dwork = to_delayed_work(work);
836 struct tegra_xudc *xudc = container_of(dwork, struct tegra_xudc,
837 plc_reset_work);
838 unsigned long flags;
839
840 spin_lock_irqsave(&xudc->lock, flags);
841
842 if (xudc->wait_csc) {
843 u32 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
844 PORTSC_PLS_SHIFT;
845
846 if (pls == PORTSC_PLS_INACTIVE) {
847 dev_info(xudc->dev, "PLS = Inactive. Toggle VBUS\n");
848 phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
849 USB_ROLE_NONE);
850 phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
851 USB_ROLE_DEVICE);
852
853 xudc->wait_csc = false;
854 }
855 }
856
857 spin_unlock_irqrestore(&xudc->lock, flags);
858}
859
860static void tegra_xudc_port_reset_war_work(struct work_struct *work)
861{
862 struct delayed_work *dwork = to_delayed_work(work);
863 struct tegra_xudc *xudc =
864 container_of(dwork, struct tegra_xudc, port_reset_war_work);
865 unsigned long flags;
866 u32 pls;
867 int ret;
868
869 spin_lock_irqsave(&xudc->lock, flags);
870
871 if (xudc->device_mode && xudc->wait_for_sec_prc) {
872 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
873 PORTSC_PLS_SHIFT;
874 dev_dbg(xudc->dev, "pls = %x\n", pls);
875
876 if (pls == PORTSC_PLS_DISABLED) {
877 dev_dbg(xudc->dev, "toggle vbus\n");
878 /* PRC doesn't complete in 100ms, toggle the vbus */
879 ret = tegra_phy_xusb_utmi_port_reset(
880 xudc->curr_utmi_phy);
881 if (ret == 1)
882 xudc->wait_for_sec_prc = 0;
883 }
884 }
885
886 spin_unlock_irqrestore(&xudc->lock, flags);
887}
888
889static dma_addr_t trb_virt_to_phys(struct tegra_xudc_ep *ep,
890 struct tegra_xudc_trb *trb)
891{
892 unsigned int index;
893
894 index = trb - ep->transfer_ring;
895
896 if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
897 return 0;
898
899 return (ep->transfer_ring_phys + index * sizeof(*trb));
900}
901
902static struct tegra_xudc_trb *trb_phys_to_virt(struct tegra_xudc_ep *ep,
903 dma_addr_t addr)
904{
905 struct tegra_xudc_trb *trb;
906 unsigned int index;
907
908 index = (addr - ep->transfer_ring_phys) / sizeof(*trb);
909
910 if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
911 return NULL;
912
913 trb = &ep->transfer_ring[index];
914
915 return trb;
916}
917
918static void ep_reload(struct tegra_xudc *xudc, unsigned int ep)
919{
920 xudc_writel(xudc, BIT(ep), EP_RELOAD);
921 xudc_readl_poll(xudc, EP_RELOAD, BIT(ep), 0);
922}
923
924static void ep_pause(struct tegra_xudc *xudc, unsigned int ep)
925{
926 u32 val;
927
928 val = xudc_readl(xudc, EP_PAUSE);
929 if (val & BIT(ep))
930 return;
931 val |= BIT(ep);
932
933 xudc_writel(xudc, val, EP_PAUSE);
934
935 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
936
937 xudc_writel(xudc, BIT(ep), EP_STCHG);
938}
939
940static void ep_unpause(struct tegra_xudc *xudc, unsigned int ep)
941{
942 u32 val;
943
944 val = xudc_readl(xudc, EP_PAUSE);
945 if (!(val & BIT(ep)))
946 return;
947 val &= ~BIT(ep);
948
949 xudc_writel(xudc, val, EP_PAUSE);
950
951 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
952
953 xudc_writel(xudc, BIT(ep), EP_STCHG);
954}
955
956static void ep_unpause_all(struct tegra_xudc *xudc)
957{
958 u32 val;
959
960 val = xudc_readl(xudc, EP_PAUSE);
961
962 xudc_writel(xudc, 0, EP_PAUSE);
963
964 xudc_readl_poll(xudc, EP_STCHG, val, val);
965
966 xudc_writel(xudc, val, EP_STCHG);
967}
968
969static void ep_halt(struct tegra_xudc *xudc, unsigned int ep)
970{
971 u32 val;
972
973 val = xudc_readl(xudc, EP_HALT);
974 if (val & BIT(ep))
975 return;
976 val |= BIT(ep);
977 xudc_writel(xudc, val, EP_HALT);
978
979 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
980
981 xudc_writel(xudc, BIT(ep), EP_STCHG);
982}
983
984static void ep_unhalt(struct tegra_xudc *xudc, unsigned int ep)
985{
986 u32 val;
987
988 val = xudc_readl(xudc, EP_HALT);
989 if (!(val & BIT(ep)))
990 return;
991 val &= ~BIT(ep);
992 xudc_writel(xudc, val, EP_HALT);
993
994 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
995
996 xudc_writel(xudc, BIT(ep), EP_STCHG);
997}
998
999static void ep_unhalt_all(struct tegra_xudc *xudc)
1000{
1001 u32 val;
1002
1003 val = xudc_readl(xudc, EP_HALT);
1004 if (!val)
1005 return;
1006 xudc_writel(xudc, 0, EP_HALT);
1007
1008 xudc_readl_poll(xudc, EP_STCHG, val, val);
1009
1010 xudc_writel(xudc, val, EP_STCHG);
1011}
1012
1013static void ep_wait_for_stopped(struct tegra_xudc *xudc, unsigned int ep)
1014{
1015 xudc_readl_poll(xudc, EP_STOPPED, BIT(ep), BIT(ep));
1016 xudc_writel(xudc, BIT(ep), EP_STOPPED);
1017}
1018
1019static void ep_wait_for_inactive(struct tegra_xudc *xudc, unsigned int ep)
1020{
1021 xudc_readl_poll(xudc, EP_THREAD_ACTIVE, BIT(ep), 0);
1022}
1023
1024static void tegra_xudc_req_done(struct tegra_xudc_ep *ep,
1025 struct tegra_xudc_request *req, int status)
1026{
1027 struct tegra_xudc *xudc = ep->xudc;
1028
1029 dev_dbg(xudc->dev, "completing request %p on EP %u with status %d\n",
1030 req, ep->index, status);
1031
1032 if (likely(req->usb_req.status == -EINPROGRESS))
1033 req->usb_req.status = status;
1034
1035 list_del_init(&req->list);
1036
1037 if (usb_endpoint_xfer_control(ep->desc)) {
1038 usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
1039 (xudc->setup_state ==
1040 DATA_STAGE_XFER));
1041 } else {
1042 usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
1043 usb_endpoint_dir_in(ep->desc));
1044 }
1045
1046 spin_unlock(&xudc->lock);
1047 usb_gadget_giveback_request(&ep->usb_ep, &req->usb_req);
1048 spin_lock(&xudc->lock);
1049}
1050
1051static void tegra_xudc_ep_nuke(struct tegra_xudc_ep *ep, int status)
1052{
1053 struct tegra_xudc_request *req;
1054
1055 while (!list_empty(&ep->queue)) {
1056 req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1057 list);
1058 tegra_xudc_req_done(ep, req, status);
1059 }
1060}
1061
1062static unsigned int ep_available_trbs(struct tegra_xudc_ep *ep)
1063{
1064 if (ep->ring_full)
1065 return 0;
1066
1067 if (ep->deq_ptr > ep->enq_ptr)
1068 return ep->deq_ptr - ep->enq_ptr - 1;
1069
1070 return XUDC_TRANSFER_RING_SIZE - (ep->enq_ptr - ep->deq_ptr) - 2;
1071}
1072
1073static void tegra_xudc_queue_one_trb(struct tegra_xudc_ep *ep,
1074 struct tegra_xudc_request *req,
1075 struct tegra_xudc_trb *trb,
1076 bool ioc)
1077{
1078 struct tegra_xudc *xudc = ep->xudc;
1079 dma_addr_t buf_addr;
1080 size_t len;
1081
1082 len = min_t(size_t, XUDC_TRB_MAX_BUFFER_SIZE, req->usb_req.length -
1083 req->buf_queued);
1084 if (len > 0)
1085 buf_addr = req->usb_req.dma + req->buf_queued;
1086 else
1087 buf_addr = 0;
1088
1089 trb_write_data_ptr(trb, buf_addr);
1090
1091 trb_write_transfer_len(trb, len);
1092 trb_write_td_size(trb, req->trbs_needed - req->trbs_queued - 1);
1093
1094 if (req->trbs_queued == req->trbs_needed - 1 ||
1095 (req->need_zlp && req->trbs_queued == req->trbs_needed - 2))
1096 trb_write_chain(trb, 0);
1097 else
1098 trb_write_chain(trb, 1);
1099
1100 trb_write_ioc(trb, ioc);
1101
1102 if (usb_endpoint_dir_out(ep->desc) ||
1103 (usb_endpoint_xfer_control(ep->desc) &&
1104 (xudc->setup_state == DATA_STAGE_RECV)))
1105 trb_write_isp(trb, 1);
1106 else
1107 trb_write_isp(trb, 0);
1108
1109 if (usb_endpoint_xfer_control(ep->desc)) {
1110 if (xudc->setup_state == DATA_STAGE_XFER ||
1111 xudc->setup_state == DATA_STAGE_RECV)
1112 trb_write_type(trb, TRB_TYPE_DATA_STAGE);
1113 else
1114 trb_write_type(trb, TRB_TYPE_STATUS_STAGE);
1115
1116 if (xudc->setup_state == DATA_STAGE_XFER ||
1117 xudc->setup_state == STATUS_STAGE_XFER)
1118 trb_write_data_stage_dir(trb, 1);
1119 else
1120 trb_write_data_stage_dir(trb, 0);
1121 } else if (usb_endpoint_xfer_isoc(ep->desc)) {
1122 trb_write_type(trb, TRB_TYPE_ISOCH);
1123 trb_write_sia(trb, 1);
1124 trb_write_frame_id(trb, 0);
1125 trb_write_tlbpc(trb, 0);
1126 } else if (usb_ss_max_streams(ep->comp_desc)) {
1127 trb_write_type(trb, TRB_TYPE_STREAM);
1128 trb_write_stream_id(trb, req->usb_req.stream_id);
1129 } else {
1130 trb_write_type(trb, TRB_TYPE_NORMAL);
1131 trb_write_stream_id(trb, 0);
1132 }
1133
1134 trb_write_cycle(trb, ep->pcs);
1135
1136 req->trbs_queued++;
1137 req->buf_queued += len;
1138
1139 dump_trb(xudc, "TRANSFER", trb);
1140}
1141
1142static unsigned int tegra_xudc_queue_trbs(struct tegra_xudc_ep *ep,
1143 struct tegra_xudc_request *req)
1144{
1145 unsigned int i, count, available;
1146 bool wait_td = false;
1147
1148 available = ep_available_trbs(ep);
1149 count = req->trbs_needed - req->trbs_queued;
1150 if (available < count) {
1151 count = available;
1152 ep->ring_full = true;
1153 }
1154
1155 /*
1156 * To generate zero-length packet on USB bus, SW needs schedule a
1157 * standalone zero-length TD. According to HW's behavior, SW needs
1158 * to schedule TDs in different ways for different endpoint types.
1159 *
1160 * For control endpoint:
1161 * - Data stage TD (IOC = 1, CH = 0)
1162 * - Ring doorbell and wait transfer event
1163 * - Data stage TD for ZLP (IOC = 1, CH = 0)
1164 * - Ring doorbell
1165 *
1166 * For bulk and interrupt endpoints:
1167 * - Normal transfer TD (IOC = 0, CH = 0)
1168 * - Normal transfer TD for ZLP (IOC = 1, CH = 0)
1169 * - Ring doorbell
1170 */
1171
1172 if (req->need_zlp && usb_endpoint_xfer_control(ep->desc) && count > 1)
1173 wait_td = true;
1174
1175 if (!req->first_trb)
1176 req->first_trb = &ep->transfer_ring[ep->enq_ptr];
1177
1178 for (i = 0; i < count; i++) {
1179 struct tegra_xudc_trb *trb = &ep->transfer_ring[ep->enq_ptr];
1180 bool ioc = false;
1181
1182 if ((i == count - 1) || (wait_td && i == count - 2))
1183 ioc = true;
1184
1185 tegra_xudc_queue_one_trb(ep, req, trb, ioc);
1186 req->last_trb = trb;
1187
1188 ep->enq_ptr++;
1189 if (ep->enq_ptr == XUDC_TRANSFER_RING_SIZE - 1) {
1190 trb = &ep->transfer_ring[ep->enq_ptr];
1191 trb_write_cycle(trb, ep->pcs);
1192 ep->pcs = !ep->pcs;
1193 ep->enq_ptr = 0;
1194 }
1195
1196 if (ioc)
1197 break;
1198 }
1199
1200 return count;
1201}
1202
1203static void tegra_xudc_ep_ring_doorbell(struct tegra_xudc_ep *ep)
1204{
1205 struct tegra_xudc *xudc = ep->xudc;
1206 u32 val;
1207
1208 if (list_empty(&ep->queue))
1209 return;
1210
1211 val = DB_TARGET(ep->index);
1212 if (usb_endpoint_xfer_control(ep->desc)) {
1213 val |= DB_STREAMID(xudc->setup_seq_num);
1214 } else if (usb_ss_max_streams(ep->comp_desc) > 0) {
1215 struct tegra_xudc_request *req;
1216
1217 /* Don't ring doorbell if the stream has been rejected. */
1218 if (ep->stream_rejected)
1219 return;
1220
1221 req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1222 list);
1223 val |= DB_STREAMID(req->usb_req.stream_id);
1224 }
1225
1226 dev_dbg(xudc->dev, "ring doorbell: %#x\n", val);
1227 xudc_writel(xudc, val, DB);
1228}
1229
1230static void tegra_xudc_ep_kick_queue(struct tegra_xudc_ep *ep)
1231{
1232 struct tegra_xudc_request *req;
1233 bool trbs_queued = false;
1234
1235 list_for_each_entry(req, &ep->queue, list) {
1236 if (ep->ring_full)
1237 break;
1238
1239 if (tegra_xudc_queue_trbs(ep, req) > 0)
1240 trbs_queued = true;
1241 }
1242
1243 if (trbs_queued)
1244 tegra_xudc_ep_ring_doorbell(ep);
1245}
1246
1247static int
1248__tegra_xudc_ep_queue(struct tegra_xudc_ep *ep, struct tegra_xudc_request *req)
1249{
1250 struct tegra_xudc *xudc = ep->xudc;
1251 int err;
1252
1253 if (usb_endpoint_xfer_control(ep->desc) && !list_empty(&ep->queue)) {
1254 dev_err(xudc->dev, "control EP has pending transfers\n");
1255 return -EINVAL;
1256 }
1257
1258 if (usb_endpoint_xfer_control(ep->desc)) {
1259 err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1260 (xudc->setup_state ==
1261 DATA_STAGE_XFER));
1262 } else {
1263 err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1264 usb_endpoint_dir_in(ep->desc));
1265 }
1266
1267 if (err < 0) {
1268 dev_err(xudc->dev, "failed to map request: %d\n", err);
1269 return err;
1270 }
1271
1272 req->first_trb = NULL;
1273 req->last_trb = NULL;
1274 req->buf_queued = 0;
1275 req->trbs_queued = 0;
1276 req->need_zlp = false;
1277 req->trbs_needed = DIV_ROUND_UP(req->usb_req.length,
1278 XUDC_TRB_MAX_BUFFER_SIZE);
1279 if (req->usb_req.length == 0)
1280 req->trbs_needed++;
1281
1282 if (!usb_endpoint_xfer_isoc(ep->desc) &&
1283 req->usb_req.zero && req->usb_req.length &&
1284 ((req->usb_req.length % ep->usb_ep.maxpacket) == 0)) {
1285 req->trbs_needed++;
1286 req->need_zlp = true;
1287 }
1288
1289 req->usb_req.status = -EINPROGRESS;
1290 req->usb_req.actual = 0;
1291
1292 list_add_tail(&req->list, &ep->queue);
1293
1294 tegra_xudc_ep_kick_queue(ep);
1295
1296 return 0;
1297}
1298
1299static int
1300tegra_xudc_ep_queue(struct usb_ep *usb_ep, struct usb_request *usb_req,
1301 gfp_t gfp)
1302{
1303 struct tegra_xudc_request *req;
1304 struct tegra_xudc_ep *ep;
1305 struct tegra_xudc *xudc;
1306 unsigned long flags;
1307 int ret;
1308
1309 if (!usb_ep || !usb_req)
1310 return -EINVAL;
1311
1312 ep = to_xudc_ep(usb_ep);
1313 req = to_xudc_req(usb_req);
1314 xudc = ep->xudc;
1315
1316 spin_lock_irqsave(&xudc->lock, flags);
1317 if (xudc->powergated || !ep->desc) {
1318 ret = -ESHUTDOWN;
1319 goto unlock;
1320 }
1321
1322 ret = __tegra_xudc_ep_queue(ep, req);
1323unlock:
1324 spin_unlock_irqrestore(&xudc->lock, flags);
1325
1326 return ret;
1327}
1328
1329static void squeeze_transfer_ring(struct tegra_xudc_ep *ep,
1330 struct tegra_xudc_request *req)
1331{
1332 struct tegra_xudc_trb *trb = req->first_trb;
1333 bool pcs_enq = trb_read_cycle(trb);
1334 bool pcs;
1335
1336 /*
1337 * Clear out all the TRBs part of or after the cancelled request,
1338 * and must correct trb cycle bit to the last un-enqueued state.
1339 */
1340 while (trb != &ep->transfer_ring[ep->enq_ptr]) {
1341 pcs = trb_read_cycle(trb);
1342 memset(trb, 0, sizeof(*trb));
1343 trb_write_cycle(trb, !pcs);
1344 trb++;
1345
1346 if (trb_read_type(trb) == TRB_TYPE_LINK)
1347 trb = ep->transfer_ring;
1348 }
1349
1350 /* Requests will be re-queued at the start of the cancelled request. */
1351 ep->enq_ptr = req->first_trb - ep->transfer_ring;
1352 /*
1353 * Retrieve the correct cycle bit state from the first trb of
1354 * the cancelled request.
1355 */
1356 ep->pcs = pcs_enq;
1357 ep->ring_full = false;
1358 list_for_each_entry_continue(req, &ep->queue, list) {
1359 req->usb_req.status = -EINPROGRESS;
1360 req->usb_req.actual = 0;
1361
1362 req->first_trb = NULL;
1363 req->last_trb = NULL;
1364 req->buf_queued = 0;
1365 req->trbs_queued = 0;
1366 }
1367}
1368
1369/*
1370 * Determine if the given TRB is in the range [first trb, last trb] for the
1371 * given request.
1372 */
1373static bool trb_in_request(struct tegra_xudc_ep *ep,
1374 struct tegra_xudc_request *req,
1375 struct tegra_xudc_trb *trb)
1376{
1377 dev_dbg(ep->xudc->dev, "%s: request %p -> %p; trb %p\n", __func__,
1378 req->first_trb, req->last_trb, trb);
1379
1380 if (trb >= req->first_trb && (trb <= req->last_trb ||
1381 req->last_trb < req->first_trb))
1382 return true;
1383
1384 if (trb < req->first_trb && trb <= req->last_trb &&
1385 req->last_trb < req->first_trb)
1386 return true;
1387
1388 return false;
1389}
1390
1391/*
1392 * Determine if the given TRB is in the range [EP enqueue pointer, first TRB)
1393 * for the given endpoint and request.
1394 */
1395static bool trb_before_request(struct tegra_xudc_ep *ep,
1396 struct tegra_xudc_request *req,
1397 struct tegra_xudc_trb *trb)
1398{
1399 struct tegra_xudc_trb *enq_trb = &ep->transfer_ring[ep->enq_ptr];
1400
1401 dev_dbg(ep->xudc->dev, "%s: request %p -> %p; enq ptr: %p; trb %p\n",
1402 __func__, req->first_trb, req->last_trb, enq_trb, trb);
1403
1404 if (trb < req->first_trb && (enq_trb <= trb ||
1405 req->first_trb < enq_trb))
1406 return true;
1407
1408 if (trb > req->first_trb && req->first_trb < enq_trb && enq_trb <= trb)
1409 return true;
1410
1411 return false;
1412}
1413
1414static int
1415__tegra_xudc_ep_dequeue(struct tegra_xudc_ep *ep,
1416 struct tegra_xudc_request *req)
1417{
1418 struct tegra_xudc *xudc = ep->xudc;
1419 struct tegra_xudc_request *r = NULL, *iter;
1420 struct tegra_xudc_trb *deq_trb;
1421 bool busy, kick_queue = false;
1422 int ret = 0;
1423
1424 /* Make sure the request is actually queued to this endpoint. */
1425 list_for_each_entry(iter, &ep->queue, list) {
1426 if (iter != req)
1427 continue;
1428 r = iter;
1429 break;
1430 }
1431
1432 if (!r)
1433 return -EINVAL;
1434
1435 /* Request hasn't been queued in the transfer ring yet. */
1436 if (!req->trbs_queued) {
1437 tegra_xudc_req_done(ep, req, -ECONNRESET);
1438 return 0;
1439 }
1440
1441 /* Halt DMA for this endpoint. */
1442 if (ep_ctx_read_state(ep->context) == EP_STATE_RUNNING) {
1443 ep_pause(xudc, ep->index);
1444 ep_wait_for_inactive(xudc, ep->index);
1445 }
1446
1447 deq_trb = trb_phys_to_virt(ep, ep_ctx_read_deq_ptr(ep->context));
1448 /* Is the hardware processing the TRB at the dequeue pointer? */
1449 busy = (trb_read_cycle(deq_trb) == ep_ctx_read_dcs(ep->context));
1450
1451 if (trb_in_request(ep, req, deq_trb) && busy) {
1452 /*
1453 * Request has been partially completed or it hasn't
1454 * started processing yet.
1455 */
1456 dma_addr_t deq_ptr;
1457
1458 squeeze_transfer_ring(ep, req);
1459
1460 req->usb_req.actual = ep_ctx_read_edtla(ep->context);
1461 tegra_xudc_req_done(ep, req, -ECONNRESET);
1462 kick_queue = true;
1463
1464 /* EDTLA is > 0: request has been partially completed */
1465 if (req->usb_req.actual > 0) {
1466 /*
1467 * Abort the pending transfer and update the dequeue
1468 * pointer
1469 */
1470 ep_ctx_write_edtla(ep->context, 0);
1471 ep_ctx_write_partial_td(ep->context, 0);
1472 ep_ctx_write_data_offset(ep->context, 0);
1473
1474 deq_ptr = trb_virt_to_phys(ep,
1475 &ep->transfer_ring[ep->enq_ptr]);
1476
1477 if (dma_mapping_error(xudc->dev, deq_ptr)) {
1478 ret = -EINVAL;
1479 } else {
1480 ep_ctx_write_deq_ptr(ep->context, deq_ptr);
1481 ep_ctx_write_dcs(ep->context, ep->pcs);
1482 ep_reload(xudc, ep->index);
1483 }
1484 }
1485 } else if (trb_before_request(ep, req, deq_trb) && busy) {
1486 /* Request hasn't started processing yet. */
1487 squeeze_transfer_ring(ep, req);
1488
1489 tegra_xudc_req_done(ep, req, -ECONNRESET);
1490 kick_queue = true;
1491 } else {
1492 /*
1493 * Request has completed, but we haven't processed the
1494 * completion event yet.
1495 */
1496 tegra_xudc_req_done(ep, req, -ECONNRESET);
1497 ret = -EINVAL;
1498 }
1499
1500 /* Resume the endpoint. */
1501 ep_unpause(xudc, ep->index);
1502
1503 if (kick_queue)
1504 tegra_xudc_ep_kick_queue(ep);
1505
1506 return ret;
1507}
1508
1509static int
1510tegra_xudc_ep_dequeue(struct usb_ep *usb_ep, struct usb_request *usb_req)
1511{
1512 struct tegra_xudc_request *req;
1513 struct tegra_xudc_ep *ep;
1514 struct tegra_xudc *xudc;
1515 unsigned long flags;
1516 int ret;
1517
1518 if (!usb_ep || !usb_req)
1519 return -EINVAL;
1520
1521 ep = to_xudc_ep(usb_ep);
1522 req = to_xudc_req(usb_req);
1523 xudc = ep->xudc;
1524
1525 spin_lock_irqsave(&xudc->lock, flags);
1526
1527 if (xudc->powergated || !ep->desc) {
1528 ret = -ESHUTDOWN;
1529 goto unlock;
1530 }
1531
1532 ret = __tegra_xudc_ep_dequeue(ep, req);
1533unlock:
1534 spin_unlock_irqrestore(&xudc->lock, flags);
1535
1536 return ret;
1537}
1538
1539static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt)
1540{
1541 struct tegra_xudc *xudc = ep->xudc;
1542
1543 if (!ep->desc)
1544 return -EINVAL;
1545
1546 if (usb_endpoint_xfer_isoc(ep->desc)) {
1547 dev_err(xudc->dev, "can't halt isochronous EP\n");
1548 return -ENOTSUPP;
1549 }
1550
1551 if (!!(xudc_readl(xudc, EP_HALT) & BIT(ep->index)) == halt) {
1552 dev_dbg(xudc->dev, "EP %u already %s\n", ep->index,
1553 halt ? "halted" : "not halted");
1554 return 0;
1555 }
1556
1557 if (halt) {
1558 ep_halt(xudc, ep->index);
1559 } else {
1560 ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1561
1562 ep_reload(xudc, ep->index);
1563
1564 ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1565 ep_ctx_write_rsvd(ep->context, 0);
1566 ep_ctx_write_partial_td(ep->context, 0);
1567 ep_ctx_write_splitxstate(ep->context, 0);
1568 ep_ctx_write_seq_num(ep->context, 0);
1569
1570 ep_reload(xudc, ep->index);
1571 ep_unpause(xudc, ep->index);
1572 ep_unhalt(xudc, ep->index);
1573
1574 tegra_xudc_ep_ring_doorbell(ep);
1575 }
1576
1577 return 0;
1578}
1579
1580static int tegra_xudc_ep_set_halt(struct usb_ep *usb_ep, int value)
1581{
1582 struct tegra_xudc_ep *ep;
1583 struct tegra_xudc *xudc;
1584 unsigned long flags;
1585 int ret;
1586
1587 if (!usb_ep)
1588 return -EINVAL;
1589
1590 ep = to_xudc_ep(usb_ep);
1591 xudc = ep->xudc;
1592
1593 spin_lock_irqsave(&xudc->lock, flags);
1594 if (xudc->powergated) {
1595 ret = -ESHUTDOWN;
1596 goto unlock;
1597 }
1598
1599 if (value && usb_endpoint_dir_in(ep->desc) &&
1600 !list_empty(&ep->queue)) {
1601 dev_err(xudc->dev, "can't halt EP with requests pending\n");
1602 ret = -EAGAIN;
1603 goto unlock;
1604 }
1605
1606 ret = __tegra_xudc_ep_set_halt(ep, value);
1607unlock:
1608 spin_unlock_irqrestore(&xudc->lock, flags);
1609
1610 return ret;
1611}
1612
1613static void tegra_xudc_ep_context_setup(struct tegra_xudc_ep *ep)
1614{
1615 const struct usb_endpoint_descriptor *desc = ep->desc;
1616 const struct usb_ss_ep_comp_descriptor *comp_desc = ep->comp_desc;
1617 struct tegra_xudc *xudc = ep->xudc;
1618 u16 maxpacket, maxburst = 0, esit = 0;
1619 u32 val;
1620
1621 maxpacket = usb_endpoint_maxp(desc);
1622 if (xudc->gadget.speed == USB_SPEED_SUPER) {
1623 if (!usb_endpoint_xfer_control(desc))
1624 maxburst = comp_desc->bMaxBurst;
1625
1626 if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc))
1627 esit = le16_to_cpu(comp_desc->wBytesPerInterval);
1628 } else if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
1629 (usb_endpoint_xfer_int(desc) ||
1630 usb_endpoint_xfer_isoc(desc))) {
1631 if (xudc->gadget.speed == USB_SPEED_HIGH) {
1632 maxburst = usb_endpoint_maxp_mult(desc) - 1;
1633 if (maxburst == 0x3) {
1634 dev_warn(xudc->dev,
1635 "invalid endpoint maxburst\n");
1636 maxburst = 0x2;
1637 }
1638 }
1639 esit = maxpacket * (maxburst + 1);
1640 }
1641
1642 memset(ep->context, 0, sizeof(*ep->context));
1643
1644 ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1645 ep_ctx_write_interval(ep->context, desc->bInterval);
1646 if (xudc->gadget.speed == USB_SPEED_SUPER) {
1647 if (usb_endpoint_xfer_isoc(desc)) {
1648 ep_ctx_write_mult(ep->context,
1649 comp_desc->bmAttributes & 0x3);
1650 }
1651
1652 if (usb_endpoint_xfer_bulk(desc)) {
1653 ep_ctx_write_max_pstreams(ep->context,
1654 comp_desc->bmAttributes &
1655 0x1f);
1656 ep_ctx_write_lsa(ep->context, 1);
1657 }
1658 }
1659
1660 if (!usb_endpoint_xfer_control(desc) && usb_endpoint_dir_out(desc))
1661 val = usb_endpoint_type(desc);
1662 else
1663 val = usb_endpoint_type(desc) + EP_TYPE_CONTROL;
1664
1665 ep_ctx_write_type(ep->context, val);
1666 ep_ctx_write_cerr(ep->context, 0x3);
1667 ep_ctx_write_max_packet_size(ep->context, maxpacket);
1668 ep_ctx_write_max_burst_size(ep->context, maxburst);
1669
1670 ep_ctx_write_deq_ptr(ep->context, ep->transfer_ring_phys);
1671 ep_ctx_write_dcs(ep->context, ep->pcs);
1672
1673 /* Select a reasonable average TRB length based on endpoint type. */
1674 switch (usb_endpoint_type(desc)) {
1675 case USB_ENDPOINT_XFER_CONTROL:
1676 val = 8;
1677 break;
1678 case USB_ENDPOINT_XFER_INT:
1679 val = 1024;
1680 break;
1681 case USB_ENDPOINT_XFER_BULK:
1682 case USB_ENDPOINT_XFER_ISOC:
1683 default:
1684 val = 3072;
1685 break;
1686 }
1687
1688 ep_ctx_write_avg_trb_len(ep->context, val);
1689 ep_ctx_write_max_esit_payload(ep->context, esit);
1690
1691 ep_ctx_write_cerrcnt(ep->context, 0x3);
1692}
1693
1694static void setup_link_trb(struct tegra_xudc_ep *ep,
1695 struct tegra_xudc_trb *trb)
1696{
1697 trb_write_data_ptr(trb, ep->transfer_ring_phys);
1698 trb_write_type(trb, TRB_TYPE_LINK);
1699 trb_write_toggle_cycle(trb, 1);
1700}
1701
1702static int __tegra_xudc_ep_disable(struct tegra_xudc_ep *ep)
1703{
1704 struct tegra_xudc *xudc = ep->xudc;
1705
1706 if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
1707 dev_err(xudc->dev, "endpoint %u already disabled\n",
1708 ep->index);
1709 return -EINVAL;
1710 }
1711
1712 ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1713
1714 ep_reload(xudc, ep->index);
1715
1716 tegra_xudc_ep_nuke(ep, -ESHUTDOWN);
1717
1718 xudc->nr_enabled_eps--;
1719 if (usb_endpoint_xfer_isoc(ep->desc))
1720 xudc->nr_isoch_eps--;
1721
1722 ep->desc = NULL;
1723 ep->comp_desc = NULL;
1724
1725 memset(ep->context, 0, sizeof(*ep->context));
1726
1727 ep_unpause(xudc, ep->index);
1728 ep_unhalt(xudc, ep->index);
1729 if (xudc_readl(xudc, EP_STOPPED) & BIT(ep->index))
1730 xudc_writel(xudc, BIT(ep->index), EP_STOPPED);
1731
1732 /*
1733 * If this is the last endpoint disabled in a de-configure request,
1734 * switch back to address state.
1735 */
1736 if ((xudc->device_state == USB_STATE_CONFIGURED) &&
1737 (xudc->nr_enabled_eps == 1)) {
1738 u32 val;
1739
1740 xudc->device_state = USB_STATE_ADDRESS;
1741 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1742
1743 val = xudc_readl(xudc, CTRL);
1744 val &= ~CTRL_RUN;
1745 xudc_writel(xudc, val, CTRL);
1746 }
1747
1748 dev_info(xudc->dev, "ep %u disabled\n", ep->index);
1749
1750 return 0;
1751}
1752
1753static int tegra_xudc_ep_disable(struct usb_ep *usb_ep)
1754{
1755 struct tegra_xudc_ep *ep;
1756 struct tegra_xudc *xudc;
1757 unsigned long flags;
1758 int ret;
1759
1760 if (!usb_ep)
1761 return -EINVAL;
1762
1763 ep = to_xudc_ep(usb_ep);
1764 xudc = ep->xudc;
1765
1766 spin_lock_irqsave(&xudc->lock, flags);
1767 if (xudc->powergated) {
1768 ret = -ESHUTDOWN;
1769 goto unlock;
1770 }
1771
1772 ret = __tegra_xudc_ep_disable(ep);
1773unlock:
1774 spin_unlock_irqrestore(&xudc->lock, flags);
1775
1776 return ret;
1777}
1778
1779static int __tegra_xudc_ep_enable(struct tegra_xudc_ep *ep,
1780 const struct usb_endpoint_descriptor *desc)
1781{
1782 struct tegra_xudc *xudc = ep->xudc;
1783 unsigned int i;
1784 u32 val;
1785
1786 if (xudc->gadget.speed == USB_SPEED_SUPER &&
1787 !usb_endpoint_xfer_control(desc) && !ep->usb_ep.comp_desc)
1788 return -EINVAL;
1789
1790 /* Disable the EP if it is not disabled */
1791 if (ep_ctx_read_state(ep->context) != EP_STATE_DISABLED)
1792 __tegra_xudc_ep_disable(ep);
1793
1794 ep->desc = desc;
1795 ep->comp_desc = ep->usb_ep.comp_desc;
1796
1797 if (usb_endpoint_xfer_isoc(desc)) {
1798 if (xudc->nr_isoch_eps > XUDC_MAX_ISOCH_EPS) {
1799 dev_err(xudc->dev, "too many isochronous endpoints\n");
1800 return -EBUSY;
1801 }
1802 xudc->nr_isoch_eps++;
1803 }
1804
1805 memset(ep->transfer_ring, 0, XUDC_TRANSFER_RING_SIZE *
1806 sizeof(*ep->transfer_ring));
1807 setup_link_trb(ep, &ep->transfer_ring[XUDC_TRANSFER_RING_SIZE - 1]);
1808
1809 ep->enq_ptr = 0;
1810 ep->deq_ptr = 0;
1811 ep->pcs = true;
1812 ep->ring_full = false;
1813 xudc->nr_enabled_eps++;
1814
1815 tegra_xudc_ep_context_setup(ep);
1816
1817 /*
1818 * No need to reload and un-halt EP0. This will be done automatically
1819 * once a valid SETUP packet is received.
1820 */
1821 if (usb_endpoint_xfer_control(desc))
1822 goto out;
1823
1824 /*
1825 * Transition to configured state once the first non-control
1826 * endpoint is enabled.
1827 */
1828 if (xudc->device_state == USB_STATE_ADDRESS) {
1829 val = xudc_readl(xudc, CTRL);
1830 val |= CTRL_RUN;
1831 xudc_writel(xudc, val, CTRL);
1832
1833 xudc->device_state = USB_STATE_CONFIGURED;
1834 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1835 }
1836
1837 if (usb_endpoint_xfer_isoc(desc)) {
1838 /*
1839 * Pause all bulk endpoints when enabling an isoch endpoint
1840 * to ensure the isoch endpoint is allocated enough bandwidth.
1841 */
1842 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1843 if (xudc->ep[i].desc &&
1844 usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1845 ep_pause(xudc, i);
1846 }
1847 }
1848
1849 ep_reload(xudc, ep->index);
1850 ep_unpause(xudc, ep->index);
1851 ep_unhalt(xudc, ep->index);
1852
1853 if (usb_endpoint_xfer_isoc(desc)) {
1854 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1855 if (xudc->ep[i].desc &&
1856 usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1857 ep_unpause(xudc, i);
1858 }
1859 }
1860
1861out:
1862 dev_info(xudc->dev, "EP %u (type: %s, dir: %s) enabled\n", ep->index,
1863 usb_ep_type_string(usb_endpoint_type(ep->desc)),
1864 usb_endpoint_dir_in(ep->desc) ? "in" : "out");
1865
1866 return 0;
1867}
1868
1869static int tegra_xudc_ep_enable(struct usb_ep *usb_ep,
1870 const struct usb_endpoint_descriptor *desc)
1871{
1872 struct tegra_xudc_ep *ep;
1873 struct tegra_xudc *xudc;
1874 unsigned long flags;
1875 int ret;
1876
1877 if (!usb_ep || !desc || (desc->bDescriptorType != USB_DT_ENDPOINT))
1878 return -EINVAL;
1879
1880 ep = to_xudc_ep(usb_ep);
1881 xudc = ep->xudc;
1882
1883 spin_lock_irqsave(&xudc->lock, flags);
1884 if (xudc->powergated) {
1885 ret = -ESHUTDOWN;
1886 goto unlock;
1887 }
1888
1889 ret = __tegra_xudc_ep_enable(ep, desc);
1890unlock:
1891 spin_unlock_irqrestore(&xudc->lock, flags);
1892
1893 return ret;
1894}
1895
1896static struct usb_request *
1897tegra_xudc_ep_alloc_request(struct usb_ep *usb_ep, gfp_t gfp)
1898{
1899 struct tegra_xudc_request *req;
1900
1901 req = kzalloc(sizeof(*req), gfp);
1902 if (!req)
1903 return NULL;
1904
1905 INIT_LIST_HEAD(&req->list);
1906
1907 return &req->usb_req;
1908}
1909
1910static void tegra_xudc_ep_free_request(struct usb_ep *usb_ep,
1911 struct usb_request *usb_req)
1912{
1913 struct tegra_xudc_request *req = to_xudc_req(usb_req);
1914
1915 kfree(req);
1916}
1917
1918static const struct usb_ep_ops tegra_xudc_ep_ops = {
1919 .enable = tegra_xudc_ep_enable,
1920 .disable = tegra_xudc_ep_disable,
1921 .alloc_request = tegra_xudc_ep_alloc_request,
1922 .free_request = tegra_xudc_ep_free_request,
1923 .queue = tegra_xudc_ep_queue,
1924 .dequeue = tegra_xudc_ep_dequeue,
1925 .set_halt = tegra_xudc_ep_set_halt,
1926};
1927
1928static int tegra_xudc_ep0_enable(struct usb_ep *usb_ep,
1929 const struct usb_endpoint_descriptor *desc)
1930{
1931 return -EBUSY;
1932}
1933
1934static int tegra_xudc_ep0_disable(struct usb_ep *usb_ep)
1935{
1936 return -EBUSY;
1937}
1938
1939static const struct usb_ep_ops tegra_xudc_ep0_ops = {
1940 .enable = tegra_xudc_ep0_enable,
1941 .disable = tegra_xudc_ep0_disable,
1942 .alloc_request = tegra_xudc_ep_alloc_request,
1943 .free_request = tegra_xudc_ep_free_request,
1944 .queue = tegra_xudc_ep_queue,
1945 .dequeue = tegra_xudc_ep_dequeue,
1946 .set_halt = tegra_xudc_ep_set_halt,
1947};
1948
1949static int tegra_xudc_gadget_get_frame(struct usb_gadget *gadget)
1950{
1951 struct tegra_xudc *xudc = to_xudc(gadget);
1952 unsigned long flags;
1953 int ret;
1954
1955 spin_lock_irqsave(&xudc->lock, flags);
1956 if (xudc->powergated) {
1957 ret = -ESHUTDOWN;
1958 goto unlock;
1959 }
1960
1961 ret = (xudc_readl(xudc, MFINDEX) & MFINDEX_FRAME_MASK) >>
1962 MFINDEX_FRAME_SHIFT;
1963unlock:
1964 spin_unlock_irqrestore(&xudc->lock, flags);
1965
1966 return ret;
1967}
1968
1969static void tegra_xudc_resume_device_state(struct tegra_xudc *xudc)
1970{
1971 unsigned int i;
1972 u32 val;
1973
1974 ep_unpause_all(xudc);
1975
1976 /* Direct link to U0. */
1977 val = xudc_readl(xudc, PORTSC);
1978 if (((val & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT) != PORTSC_PLS_U0) {
1979 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
1980 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
1981 xudc_writel(xudc, val, PORTSC);
1982 }
1983
1984 if (xudc->device_state == USB_STATE_SUSPENDED) {
1985 xudc->device_state = xudc->resume_state;
1986 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1987 xudc->resume_state = 0;
1988 }
1989
1990 /*
1991 * Doorbells may be dropped if they are sent too soon (< ~200ns)
1992 * after unpausing the endpoint. Wait for 500ns just to be safe.
1993 */
1994 ndelay(500);
1995 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
1996 tegra_xudc_ep_ring_doorbell(&xudc->ep[i]);
1997}
1998
1999static int tegra_xudc_gadget_wakeup(struct usb_gadget *gadget)
2000{
2001 struct tegra_xudc *xudc = to_xudc(gadget);
2002 unsigned long flags;
2003 int ret = 0;
2004 u32 val;
2005
2006 spin_lock_irqsave(&xudc->lock, flags);
2007
2008 if (xudc->powergated) {
2009 ret = -ESHUTDOWN;
2010 goto unlock;
2011 }
2012 val = xudc_readl(xudc, PORTPM);
2013 dev_dbg(xudc->dev, "%s: PORTPM=%#x, speed=%x\n", __func__,
2014 val, gadget->speed);
2015
2016 if (((xudc->gadget.speed <= USB_SPEED_HIGH) &&
2017 (val & PORTPM_RWE)) ||
2018 ((xudc->gadget.speed == USB_SPEED_SUPER) &&
2019 (val & PORTPM_FRWE))) {
2020 tegra_xudc_resume_device_state(xudc);
2021
2022 /* Send Device Notification packet. */
2023 if (xudc->gadget.speed == USB_SPEED_SUPER) {
2024 val = DEVNOTIF_LO_TYPE(DEVNOTIF_LO_TYPE_FUNCTION_WAKE)
2025 | DEVNOTIF_LO_TRIG;
2026 xudc_writel(xudc, 0, DEVNOTIF_HI);
2027 xudc_writel(xudc, val, DEVNOTIF_LO);
2028 }
2029 }
2030
2031unlock:
2032 dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2033 spin_unlock_irqrestore(&xudc->lock, flags);
2034
2035 return ret;
2036}
2037
2038static int tegra_xudc_gadget_pullup(struct usb_gadget *gadget, int is_on)
2039{
2040 struct tegra_xudc *xudc = to_xudc(gadget);
2041 unsigned long flags;
2042 u32 val;
2043
2044 pm_runtime_get_sync(xudc->dev);
2045
2046 spin_lock_irqsave(&xudc->lock, flags);
2047
2048 if (is_on != xudc->pullup) {
2049 val = xudc_readl(xudc, CTRL);
2050 if (is_on)
2051 val |= CTRL_ENABLE;
2052 else
2053 val &= ~CTRL_ENABLE;
2054 xudc_writel(xudc, val, CTRL);
2055 }
2056
2057 xudc->pullup = is_on;
2058 dev_dbg(xudc->dev, "%s: pullup:%d", __func__, is_on);
2059
2060 spin_unlock_irqrestore(&xudc->lock, flags);
2061
2062 pm_runtime_put(xudc->dev);
2063
2064 return 0;
2065}
2066
2067static int tegra_xudc_gadget_start(struct usb_gadget *gadget,
2068 struct usb_gadget_driver *driver)
2069{
2070 struct tegra_xudc *xudc = to_xudc(gadget);
2071 unsigned long flags;
2072 u32 val;
2073 int ret;
2074 unsigned int i;
2075
2076 if (!driver)
2077 return -EINVAL;
2078
2079 pm_runtime_get_sync(xudc->dev);
2080
2081 spin_lock_irqsave(&xudc->lock, flags);
2082
2083 if (xudc->driver) {
2084 ret = -EBUSY;
2085 goto unlock;
2086 }
2087
2088 xudc->setup_state = WAIT_FOR_SETUP;
2089 xudc->device_state = USB_STATE_DEFAULT;
2090 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2091
2092 ret = __tegra_xudc_ep_enable(&xudc->ep[0], &tegra_xudc_ep0_desc);
2093 if (ret < 0)
2094 goto unlock;
2095
2096 val = xudc_readl(xudc, CTRL);
2097 val |= CTRL_IE | CTRL_LSE;
2098 xudc_writel(xudc, val, CTRL);
2099
2100 val = xudc_readl(xudc, PORTHALT);
2101 val |= PORTHALT_STCHG_INTR_EN;
2102 xudc_writel(xudc, val, PORTHALT);
2103
2104 if (xudc->pullup) {
2105 val = xudc_readl(xudc, CTRL);
2106 val |= CTRL_ENABLE;
2107 xudc_writel(xudc, val, CTRL);
2108 }
2109
2110 for (i = 0; i < xudc->soc->num_phys; i++)
2111 if (xudc->usbphy[i])
2112 otg_set_peripheral(xudc->usbphy[i]->otg, gadget);
2113
2114 xudc->driver = driver;
2115unlock:
2116 dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2117 spin_unlock_irqrestore(&xudc->lock, flags);
2118
2119 pm_runtime_put(xudc->dev);
2120
2121 return ret;
2122}
2123
2124static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
2125{
2126 struct tegra_xudc *xudc = to_xudc(gadget);
2127 unsigned long flags;
2128 u32 val;
2129 unsigned int i;
2130
2131 pm_runtime_get_sync(xudc->dev);
2132
2133 spin_lock_irqsave(&xudc->lock, flags);
2134
2135 for (i = 0; i < xudc->soc->num_phys; i++)
2136 if (xudc->usbphy[i])
2137 otg_set_peripheral(xudc->usbphy[i]->otg, NULL);
2138
2139 val = xudc_readl(xudc, CTRL);
2140 val &= ~(CTRL_IE | CTRL_ENABLE);
2141 xudc_writel(xudc, val, CTRL);
2142
2143 __tegra_xudc_ep_disable(&xudc->ep[0]);
2144
2145 xudc->driver = NULL;
2146 dev_dbg(xudc->dev, "Gadget stopped");
2147
2148 spin_unlock_irqrestore(&xudc->lock, flags);
2149
2150 pm_runtime_put(xudc->dev);
2151
2152 return 0;
2153}
2154
2155static int tegra_xudc_gadget_vbus_draw(struct usb_gadget *gadget,
2156 unsigned int m_a)
2157{
2158 int ret = 0;
2159 struct tegra_xudc *xudc = to_xudc(gadget);
2160
2161 dev_dbg(xudc->dev, "%s: %u mA\n", __func__, m_a);
2162
2163 if (xudc->curr_usbphy->chg_type == SDP_TYPE)
2164 ret = usb_phy_set_power(xudc->curr_usbphy, m_a);
2165
2166 return ret;
2167}
2168
2169static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
2170{
2171 struct tegra_xudc *xudc = to_xudc(gadget);
2172
2173 dev_dbg(xudc->dev, "%s: %d\n", __func__, is_on);
2174 xudc->selfpowered = !!is_on;
2175
2176 return 0;
2177}
2178
2179static const struct usb_gadget_ops tegra_xudc_gadget_ops = {
2180 .get_frame = tegra_xudc_gadget_get_frame,
2181 .wakeup = tegra_xudc_gadget_wakeup,
2182 .pullup = tegra_xudc_gadget_pullup,
2183 .udc_start = tegra_xudc_gadget_start,
2184 .udc_stop = tegra_xudc_gadget_stop,
2185 .vbus_draw = tegra_xudc_gadget_vbus_draw,
2186 .set_selfpowered = tegra_xudc_set_selfpowered,
2187};
2188
2189static void no_op_complete(struct usb_ep *ep, struct usb_request *req)
2190{
2191}
2192
2193static int
2194tegra_xudc_ep0_queue_status(struct tegra_xudc *xudc,
2195 void (*cmpl)(struct usb_ep *, struct usb_request *))
2196{
2197 xudc->ep0_req->usb_req.buf = NULL;
2198 xudc->ep0_req->usb_req.dma = 0;
2199 xudc->ep0_req->usb_req.length = 0;
2200 xudc->ep0_req->usb_req.complete = cmpl;
2201 xudc->ep0_req->usb_req.context = xudc;
2202
2203 return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2204}
2205
2206static int
2207tegra_xudc_ep0_queue_data(struct tegra_xudc *xudc, void *buf, size_t len,
2208 void (*cmpl)(struct usb_ep *, struct usb_request *))
2209{
2210 xudc->ep0_req->usb_req.buf = buf;
2211 xudc->ep0_req->usb_req.length = len;
2212 xudc->ep0_req->usb_req.complete = cmpl;
2213 xudc->ep0_req->usb_req.context = xudc;
2214
2215 return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2216}
2217
2218static void tegra_xudc_ep0_req_done(struct tegra_xudc *xudc)
2219{
2220 switch (xudc->setup_state) {
2221 case DATA_STAGE_XFER:
2222 xudc->setup_state = STATUS_STAGE_RECV;
2223 tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2224 break;
2225 case DATA_STAGE_RECV:
2226 xudc->setup_state = STATUS_STAGE_XFER;
2227 tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2228 break;
2229 default:
2230 xudc->setup_state = WAIT_FOR_SETUP;
2231 break;
2232 }
2233}
2234
2235static int tegra_xudc_ep0_delegate_req(struct tegra_xudc *xudc,
2236 struct usb_ctrlrequest *ctrl)
2237{
2238 int ret;
2239
2240 spin_unlock(&xudc->lock);
2241 ret = xudc->driver->setup(&xudc->gadget, ctrl);
2242 spin_lock(&xudc->lock);
2243
2244 return ret;
2245}
2246
2247static void set_feature_complete(struct usb_ep *ep, struct usb_request *req)
2248{
2249 struct tegra_xudc *xudc = req->context;
2250
2251 if (xudc->test_mode_pattern) {
2252 xudc_writel(xudc, xudc->test_mode_pattern, PORT_TM);
2253 xudc->test_mode_pattern = 0;
2254 }
2255}
2256
2257static int tegra_xudc_ep0_set_feature(struct tegra_xudc *xudc,
2258 struct usb_ctrlrequest *ctrl)
2259{
2260 bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
2261 u32 feature = le16_to_cpu(ctrl->wValue);
2262 u32 index = le16_to_cpu(ctrl->wIndex);
2263 u32 val, ep;
2264 int ret;
2265
2266 if (le16_to_cpu(ctrl->wLength) != 0)
2267 return -EINVAL;
2268
2269 switch (ctrl->bRequestType & USB_RECIP_MASK) {
2270 case USB_RECIP_DEVICE:
2271 switch (feature) {
2272 case USB_DEVICE_REMOTE_WAKEUP:
2273 if ((xudc->gadget.speed == USB_SPEED_SUPER) ||
2274 (xudc->device_state == USB_STATE_DEFAULT))
2275 return -EINVAL;
2276
2277 val = xudc_readl(xudc, PORTPM);
2278 if (set)
2279 val |= PORTPM_RWE;
2280 else
2281 val &= ~PORTPM_RWE;
2282
2283 xudc_writel(xudc, val, PORTPM);
2284 break;
2285 case USB_DEVICE_U1_ENABLE:
2286 case USB_DEVICE_U2_ENABLE:
2287 if ((xudc->device_state != USB_STATE_CONFIGURED) ||
2288 (xudc->gadget.speed != USB_SPEED_SUPER))
2289 return -EINVAL;
2290
2291 val = xudc_readl(xudc, PORTPM);
2292 if ((feature == USB_DEVICE_U1_ENABLE) &&
2293 xudc->soc->u1_enable) {
2294 if (set)
2295 val |= PORTPM_U1E;
2296 else
2297 val &= ~PORTPM_U1E;
2298 }
2299
2300 if ((feature == USB_DEVICE_U2_ENABLE) &&
2301 xudc->soc->u2_enable) {
2302 if (set)
2303 val |= PORTPM_U2E;
2304 else
2305 val &= ~PORTPM_U2E;
2306 }
2307
2308 xudc_writel(xudc, val, PORTPM);
2309 break;
2310 case USB_DEVICE_TEST_MODE:
2311 if (xudc->gadget.speed != USB_SPEED_HIGH)
2312 return -EINVAL;
2313
2314 if (!set)
2315 return -EINVAL;
2316
2317 xudc->test_mode_pattern = index >> 8;
2318 break;
2319 default:
2320 return -EINVAL;
2321 }
2322
2323 break;
2324 case USB_RECIP_INTERFACE:
2325 if (xudc->device_state != USB_STATE_CONFIGURED)
2326 return -EINVAL;
2327
2328 switch (feature) {
2329 case USB_INTRF_FUNC_SUSPEND:
2330 if (set) {
2331 val = xudc_readl(xudc, PORTPM);
2332
2333 if (index & USB_INTRF_FUNC_SUSPEND_RW)
2334 val |= PORTPM_FRWE;
2335 else
2336 val &= ~PORTPM_FRWE;
2337
2338 xudc_writel(xudc, val, PORTPM);
2339 }
2340
2341 return tegra_xudc_ep0_delegate_req(xudc, ctrl);
2342 default:
2343 return -EINVAL;
2344 }
2345
2346 break;
2347 case USB_RECIP_ENDPOINT:
2348 ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2349 ((index & USB_DIR_IN) ? 1 : 0);
2350
2351 if ((xudc->device_state == USB_STATE_DEFAULT) ||
2352 ((xudc->device_state == USB_STATE_ADDRESS) &&
2353 (index != 0)))
2354 return -EINVAL;
2355
2356 ret = __tegra_xudc_ep_set_halt(&xudc->ep[ep], set);
2357 if (ret < 0)
2358 return ret;
2359 break;
2360 default:
2361 return -EINVAL;
2362 }
2363
2364 return tegra_xudc_ep0_queue_status(xudc, set_feature_complete);
2365}
2366
2367static int tegra_xudc_ep0_get_status(struct tegra_xudc *xudc,
2368 struct usb_ctrlrequest *ctrl)
2369{
2370 struct tegra_xudc_ep_context *ep_ctx;
2371 u32 val, ep, index = le16_to_cpu(ctrl->wIndex);
2372 u16 status = 0;
2373
2374 if (!(ctrl->bRequestType & USB_DIR_IN))
2375 return -EINVAL;
2376
2377 if ((le16_to_cpu(ctrl->wValue) != 0) ||
2378 (le16_to_cpu(ctrl->wLength) != 2))
2379 return -EINVAL;
2380
2381 switch (ctrl->bRequestType & USB_RECIP_MASK) {
2382 case USB_RECIP_DEVICE:
2383 val = xudc_readl(xudc, PORTPM);
2384
2385 if (xudc->selfpowered)
2386 status |= BIT(USB_DEVICE_SELF_POWERED);
2387
2388 if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
2389 (val & PORTPM_RWE))
2390 status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
2391
2392 if (xudc->gadget.speed == USB_SPEED_SUPER) {
2393 if (val & PORTPM_U1E)
2394 status |= BIT(USB_DEV_STAT_U1_ENABLED);
2395 if (val & PORTPM_U2E)
2396 status |= BIT(USB_DEV_STAT_U2_ENABLED);
2397 }
2398 break;
2399 case USB_RECIP_INTERFACE:
2400 if (xudc->gadget.speed == USB_SPEED_SUPER) {
2401 status |= USB_INTRF_STAT_FUNC_RW_CAP;
2402 val = xudc_readl(xudc, PORTPM);
2403 if (val & PORTPM_FRWE)
2404 status |= USB_INTRF_STAT_FUNC_RW;
2405 }
2406 break;
2407 case USB_RECIP_ENDPOINT:
2408 ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2409 ((index & USB_DIR_IN) ? 1 : 0);
2410 ep_ctx = &xudc->ep_context[ep];
2411
2412 if ((xudc->device_state != USB_STATE_CONFIGURED) &&
2413 ((xudc->device_state != USB_STATE_ADDRESS) || (ep != 0)))
2414 return -EINVAL;
2415
2416 if (ep_ctx_read_state(ep_ctx) == EP_STATE_DISABLED)
2417 return -EINVAL;
2418
2419 if (xudc_readl(xudc, EP_HALT) & BIT(ep))
2420 status |= BIT(USB_ENDPOINT_HALT);
2421 break;
2422 default:
2423 return -EINVAL;
2424 }
2425
2426 xudc->status_buf = cpu_to_le16(status);
2427 return tegra_xudc_ep0_queue_data(xudc, &xudc->status_buf,
2428 sizeof(xudc->status_buf),
2429 no_op_complete);
2430}
2431
2432static void set_sel_complete(struct usb_ep *ep, struct usb_request *req)
2433{
2434 /* Nothing to do with SEL values */
2435}
2436
2437static int tegra_xudc_ep0_set_sel(struct tegra_xudc *xudc,
2438 struct usb_ctrlrequest *ctrl)
2439{
2440 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2441 USB_TYPE_STANDARD))
2442 return -EINVAL;
2443
2444 if (xudc->device_state == USB_STATE_DEFAULT)
2445 return -EINVAL;
2446
2447 if ((le16_to_cpu(ctrl->wIndex) != 0) ||
2448 (le16_to_cpu(ctrl->wValue) != 0) ||
2449 (le16_to_cpu(ctrl->wLength) != 6))
2450 return -EINVAL;
2451
2452 return tegra_xudc_ep0_queue_data(xudc, &xudc->sel_timing,
2453 sizeof(xudc->sel_timing),
2454 set_sel_complete);
2455}
2456
2457static void set_isoch_delay_complete(struct usb_ep *ep, struct usb_request *req)
2458{
2459 /* Nothing to do with isoch delay */
2460}
2461
2462static int tegra_xudc_ep0_set_isoch_delay(struct tegra_xudc *xudc,
2463 struct usb_ctrlrequest *ctrl)
2464{
2465 u32 delay = le16_to_cpu(ctrl->wValue);
2466
2467 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2468 USB_TYPE_STANDARD))
2469 return -EINVAL;
2470
2471 if ((delay > 65535) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2472 (le16_to_cpu(ctrl->wLength) != 0))
2473 return -EINVAL;
2474
2475 xudc->isoch_delay = delay;
2476
2477 return tegra_xudc_ep0_queue_status(xudc, set_isoch_delay_complete);
2478}
2479
2480static void set_address_complete(struct usb_ep *ep, struct usb_request *req)
2481{
2482 struct tegra_xudc *xudc = req->context;
2483
2484 if ((xudc->device_state == USB_STATE_DEFAULT) &&
2485 (xudc->dev_addr != 0)) {
2486 xudc->device_state = USB_STATE_ADDRESS;
2487 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2488 } else if ((xudc->device_state == USB_STATE_ADDRESS) &&
2489 (xudc->dev_addr == 0)) {
2490 xudc->device_state = USB_STATE_DEFAULT;
2491 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2492 }
2493}
2494
2495static int tegra_xudc_ep0_set_address(struct tegra_xudc *xudc,
2496 struct usb_ctrlrequest *ctrl)
2497{
2498 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2499 u32 val, addr = le16_to_cpu(ctrl->wValue);
2500
2501 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2502 USB_TYPE_STANDARD))
2503 return -EINVAL;
2504
2505 if ((addr > 127) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2506 (le16_to_cpu(ctrl->wLength) != 0))
2507 return -EINVAL;
2508
2509 if (xudc->device_state == USB_STATE_CONFIGURED)
2510 return -EINVAL;
2511
2512 dev_dbg(xudc->dev, "set address: %u\n", addr);
2513
2514 xudc->dev_addr = addr;
2515 val = xudc_readl(xudc, CTRL);
2516 val &= ~(CTRL_DEVADDR_MASK);
2517 val |= CTRL_DEVADDR(addr);
2518 xudc_writel(xudc, val, CTRL);
2519
2520 ep_ctx_write_devaddr(ep0->context, addr);
2521
2522 return tegra_xudc_ep0_queue_status(xudc, set_address_complete);
2523}
2524
2525static int tegra_xudc_ep0_standard_req(struct tegra_xudc *xudc,
2526 struct usb_ctrlrequest *ctrl)
2527{
2528 int ret;
2529
2530 switch (ctrl->bRequest) {
2531 case USB_REQ_GET_STATUS:
2532 dev_dbg(xudc->dev, "USB_REQ_GET_STATUS\n");
2533 ret = tegra_xudc_ep0_get_status(xudc, ctrl);
2534 break;
2535 case USB_REQ_SET_ADDRESS:
2536 dev_dbg(xudc->dev, "USB_REQ_SET_ADDRESS\n");
2537 ret = tegra_xudc_ep0_set_address(xudc, ctrl);
2538 break;
2539 case USB_REQ_SET_SEL:
2540 dev_dbg(xudc->dev, "USB_REQ_SET_SEL\n");
2541 ret = tegra_xudc_ep0_set_sel(xudc, ctrl);
2542 break;
2543 case USB_REQ_SET_ISOCH_DELAY:
2544 dev_dbg(xudc->dev, "USB_REQ_SET_ISOCH_DELAY\n");
2545 ret = tegra_xudc_ep0_set_isoch_delay(xudc, ctrl);
2546 break;
2547 case USB_REQ_CLEAR_FEATURE:
2548 case USB_REQ_SET_FEATURE:
2549 dev_dbg(xudc->dev, "USB_REQ_CLEAR/SET_FEATURE\n");
2550 ret = tegra_xudc_ep0_set_feature(xudc, ctrl);
2551 break;
2552 case USB_REQ_SET_CONFIGURATION:
2553 dev_dbg(xudc->dev, "USB_REQ_SET_CONFIGURATION\n");
2554 /*
2555 * In theory we need to clear RUN bit before status stage of
2556 * deconfig request sent, but this seems to be causing problems.
2557 * Clear RUN once all endpoints are disabled instead.
2558 */
2559 fallthrough;
2560 default:
2561 ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2562 break;
2563 }
2564
2565 return ret;
2566}
2567
2568static void tegra_xudc_handle_ep0_setup_packet(struct tegra_xudc *xudc,
2569 struct usb_ctrlrequest *ctrl,
2570 u16 seq_num)
2571{
2572 int ret;
2573
2574 xudc->setup_seq_num = seq_num;
2575
2576 /* Ensure EP0 is unhalted. */
2577 ep_unhalt(xudc, 0);
2578
2579 /*
2580 * On Tegra210, setup packets with sequence numbers 0xfffe or 0xffff
2581 * are invalid. Halt EP0 until we get a valid packet.
2582 */
2583 if (xudc->soc->invalid_seq_num &&
2584 (seq_num == 0xfffe || seq_num == 0xffff)) {
2585 dev_warn(xudc->dev, "invalid sequence number detected\n");
2586 ep_halt(xudc, 0);
2587 return;
2588 }
2589
2590 if (ctrl->wLength)
2591 xudc->setup_state = (ctrl->bRequestType & USB_DIR_IN) ?
2592 DATA_STAGE_XFER : DATA_STAGE_RECV;
2593 else
2594 xudc->setup_state = STATUS_STAGE_XFER;
2595
2596 if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
2597 ret = tegra_xudc_ep0_standard_req(xudc, ctrl);
2598 else
2599 ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2600
2601 if (ret < 0) {
2602 dev_warn(xudc->dev, "setup request failed: %d\n", ret);
2603 xudc->setup_state = WAIT_FOR_SETUP;
2604 ep_halt(xudc, 0);
2605 }
2606}
2607
2608static void tegra_xudc_handle_ep0_event(struct tegra_xudc *xudc,
2609 struct tegra_xudc_trb *event)
2610{
2611 struct usb_ctrlrequest *ctrl = (struct usb_ctrlrequest *)event;
2612 u16 seq_num = trb_read_seq_num(event);
2613
2614 if (xudc->setup_state != WAIT_FOR_SETUP) {
2615 /*
2616 * The controller is in the process of handling another
2617 * setup request. Queue subsequent requests and handle
2618 * the last one once the controller reports a sequence
2619 * number error.
2620 */
2621 memcpy(&xudc->setup_packet.ctrl_req, ctrl, sizeof(*ctrl));
2622 xudc->setup_packet.seq_num = seq_num;
2623 xudc->queued_setup_packet = true;
2624 } else {
2625 tegra_xudc_handle_ep0_setup_packet(xudc, ctrl, seq_num);
2626 }
2627}
2628
2629static struct tegra_xudc_request *
2630trb_to_request(struct tegra_xudc_ep *ep, struct tegra_xudc_trb *trb)
2631{
2632 struct tegra_xudc_request *req;
2633
2634 list_for_each_entry(req, &ep->queue, list) {
2635 if (!req->trbs_queued)
2636 break;
2637
2638 if (trb_in_request(ep, req, trb))
2639 return req;
2640 }
2641
2642 return NULL;
2643}
2644
2645static void tegra_xudc_handle_transfer_completion(struct tegra_xudc *xudc,
2646 struct tegra_xudc_ep *ep,
2647 struct tegra_xudc_trb *event)
2648{
2649 struct tegra_xudc_request *req;
2650 struct tegra_xudc_trb *trb;
2651 bool short_packet;
2652
2653 short_packet = (trb_read_cmpl_code(event) ==
2654 TRB_CMPL_CODE_SHORT_PACKET);
2655
2656 trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2657 req = trb_to_request(ep, trb);
2658
2659 /*
2660 * TDs are complete on short packet or when the completed TRB is the
2661 * last TRB in the TD (the CHAIN bit is unset).
2662 */
2663 if (req && (short_packet || (!trb_read_chain(trb) &&
2664 (req->trbs_needed == req->trbs_queued)))) {
2665 struct tegra_xudc_trb *last = req->last_trb;
2666 unsigned int residual;
2667
2668 residual = trb_read_transfer_len(event);
2669 req->usb_req.actual = req->usb_req.length - residual;
2670
2671 dev_dbg(xudc->dev, "bytes transferred %u / %u\n",
2672 req->usb_req.actual, req->usb_req.length);
2673
2674 tegra_xudc_req_done(ep, req, 0);
2675
2676 if (ep->desc && usb_endpoint_xfer_control(ep->desc))
2677 tegra_xudc_ep0_req_done(xudc);
2678
2679 /*
2680 * Advance the dequeue pointer past the end of the current TD
2681 * on short packet completion.
2682 */
2683 if (short_packet) {
2684 ep->deq_ptr = (last - ep->transfer_ring) + 1;
2685 if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2686 ep->deq_ptr = 0;
2687 }
2688 } else if (!req) {
2689 dev_warn(xudc->dev, "transfer event on dequeued request\n");
2690 }
2691
2692 if (ep->desc)
2693 tegra_xudc_ep_kick_queue(ep);
2694}
2695
2696static void tegra_xudc_handle_transfer_event(struct tegra_xudc *xudc,
2697 struct tegra_xudc_trb *event)
2698{
2699 unsigned int ep_index = trb_read_endpoint_id(event);
2700 struct tegra_xudc_ep *ep = &xudc->ep[ep_index];
2701 struct tegra_xudc_trb *trb;
2702 u16 comp_code;
2703
2704 if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
2705 dev_warn(xudc->dev, "transfer event on disabled EP %u\n",
2706 ep_index);
2707 return;
2708 }
2709
2710 /* Update transfer ring dequeue pointer. */
2711 trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2712 comp_code = trb_read_cmpl_code(event);
2713 if (comp_code != TRB_CMPL_CODE_BABBLE_DETECTED_ERR) {
2714 ep->deq_ptr = (trb - ep->transfer_ring) + 1;
2715
2716 if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2717 ep->deq_ptr = 0;
2718 ep->ring_full = false;
2719 }
2720
2721 switch (comp_code) {
2722 case TRB_CMPL_CODE_SUCCESS:
2723 case TRB_CMPL_CODE_SHORT_PACKET:
2724 tegra_xudc_handle_transfer_completion(xudc, ep, event);
2725 break;
2726 case TRB_CMPL_CODE_HOST_REJECTED:
2727 dev_info(xudc->dev, "stream rejected on EP %u\n", ep_index);
2728
2729 ep->stream_rejected = true;
2730 break;
2731 case TRB_CMPL_CODE_PRIME_PIPE_RECEIVED:
2732 dev_info(xudc->dev, "prime pipe received on EP %u\n", ep_index);
2733
2734 if (ep->stream_rejected) {
2735 ep->stream_rejected = false;
2736 /*
2737 * An EP is stopped when a stream is rejected. Wait
2738 * for the EP to report that it is stopped and then
2739 * un-stop it.
2740 */
2741 ep_wait_for_stopped(xudc, ep_index);
2742 }
2743 tegra_xudc_ep_ring_doorbell(ep);
2744 break;
2745 case TRB_CMPL_CODE_BABBLE_DETECTED_ERR:
2746 /*
2747 * Wait for the EP to be stopped so the controller stops
2748 * processing doorbells.
2749 */
2750 ep_wait_for_stopped(xudc, ep_index);
2751 ep->enq_ptr = ep->deq_ptr;
2752 tegra_xudc_ep_nuke(ep, -EIO);
2753 fallthrough;
2754 case TRB_CMPL_CODE_STREAM_NUMP_ERROR:
2755 case TRB_CMPL_CODE_CTRL_DIR_ERR:
2756 case TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR:
2757 case TRB_CMPL_CODE_RING_UNDERRUN:
2758 case TRB_CMPL_CODE_RING_OVERRUN:
2759 case TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN:
2760 case TRB_CMPL_CODE_USB_TRANS_ERR:
2761 case TRB_CMPL_CODE_TRB_ERR:
2762 dev_err(xudc->dev, "completion error %#x on EP %u\n",
2763 comp_code, ep_index);
2764
2765 ep_halt(xudc, ep_index);
2766 break;
2767 case TRB_CMPL_CODE_CTRL_SEQNUM_ERR:
2768 dev_info(xudc->dev, "sequence number error\n");
2769
2770 /*
2771 * Kill any queued control request and skip to the last
2772 * setup packet we received.
2773 */
2774 tegra_xudc_ep_nuke(ep, -EINVAL);
2775 xudc->setup_state = WAIT_FOR_SETUP;
2776 if (!xudc->queued_setup_packet)
2777 break;
2778
2779 tegra_xudc_handle_ep0_setup_packet(xudc,
2780 &xudc->setup_packet.ctrl_req,
2781 xudc->setup_packet.seq_num);
2782 xudc->queued_setup_packet = false;
2783 break;
2784 case TRB_CMPL_CODE_STOPPED:
2785 dev_dbg(xudc->dev, "stop completion code on EP %u\n",
2786 ep_index);
2787
2788 /* Disconnected. */
2789 tegra_xudc_ep_nuke(ep, -ECONNREFUSED);
2790 break;
2791 default:
2792 dev_dbg(xudc->dev, "completion event %#x on EP %u\n",
2793 comp_code, ep_index);
2794 break;
2795 }
2796}
2797
2798static void tegra_xudc_reset(struct tegra_xudc *xudc)
2799{
2800 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2801 dma_addr_t deq_ptr;
2802 unsigned int i;
2803
2804 xudc->setup_state = WAIT_FOR_SETUP;
2805 xudc->device_state = USB_STATE_DEFAULT;
2806 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2807
2808 ep_unpause_all(xudc);
2809
2810 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2811 tegra_xudc_ep_nuke(&xudc->ep[i], -ESHUTDOWN);
2812
2813 /*
2814 * Reset sequence number and dequeue pointer to flush the transfer
2815 * ring.
2816 */
2817 ep0->deq_ptr = ep0->enq_ptr;
2818 ep0->ring_full = false;
2819
2820 xudc->setup_seq_num = 0;
2821 xudc->queued_setup_packet = false;
2822
2823 ep_ctx_write_rsvd(ep0->context, 0);
2824 ep_ctx_write_partial_td(ep0->context, 0);
2825 ep_ctx_write_splitxstate(ep0->context, 0);
2826 ep_ctx_write_seq_num(ep0->context, 0);
2827
2828 deq_ptr = trb_virt_to_phys(ep0, &ep0->transfer_ring[ep0->deq_ptr]);
2829
2830 if (!dma_mapping_error(xudc->dev, deq_ptr)) {
2831 ep_ctx_write_deq_ptr(ep0->context, deq_ptr);
2832 ep_ctx_write_dcs(ep0->context, ep0->pcs);
2833 }
2834
2835 ep_unhalt_all(xudc);
2836 ep_reload(xudc, 0);
2837 ep_unpause(xudc, 0);
2838}
2839
2840static void tegra_xudc_port_connect(struct tegra_xudc *xudc)
2841{
2842 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2843 u16 maxpacket;
2844 u32 val;
2845
2846 val = (xudc_readl(xudc, PORTSC) & PORTSC_PS_MASK) >> PORTSC_PS_SHIFT;
2847 switch (val) {
2848 case PORTSC_PS_LS:
2849 xudc->gadget.speed = USB_SPEED_LOW;
2850 break;
2851 case PORTSC_PS_FS:
2852 xudc->gadget.speed = USB_SPEED_FULL;
2853 break;
2854 case PORTSC_PS_HS:
2855 xudc->gadget.speed = USB_SPEED_HIGH;
2856 break;
2857 case PORTSC_PS_SS:
2858 xudc->gadget.speed = USB_SPEED_SUPER;
2859 break;
2860 default:
2861 xudc->gadget.speed = USB_SPEED_UNKNOWN;
2862 break;
2863 }
2864
2865 xudc->device_state = USB_STATE_DEFAULT;
2866 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2867
2868 xudc->setup_state = WAIT_FOR_SETUP;
2869
2870 if (xudc->gadget.speed == USB_SPEED_SUPER)
2871 maxpacket = 512;
2872 else
2873 maxpacket = 64;
2874
2875 ep_ctx_write_max_packet_size(ep0->context, maxpacket);
2876 tegra_xudc_ep0_desc.wMaxPacketSize = cpu_to_le16(maxpacket);
2877 usb_ep_set_maxpacket_limit(&ep0->usb_ep, maxpacket);
2878
2879 if (!xudc->soc->u1_enable) {
2880 val = xudc_readl(xudc, PORTPM);
2881 val &= ~(PORTPM_U1TIMEOUT_MASK);
2882 xudc_writel(xudc, val, PORTPM);
2883 }
2884
2885 if (!xudc->soc->u2_enable) {
2886 val = xudc_readl(xudc, PORTPM);
2887 val &= ~(PORTPM_U2TIMEOUT_MASK);
2888 xudc_writel(xudc, val, PORTPM);
2889 }
2890
2891 if (xudc->gadget.speed <= USB_SPEED_HIGH) {
2892 val = xudc_readl(xudc, PORTPM);
2893 val &= ~(PORTPM_L1S_MASK);
2894 if (xudc->soc->lpm_enable)
2895 val |= PORTPM_L1S(PORTPM_L1S_ACCEPT);
2896 else
2897 val |= PORTPM_L1S(PORTPM_L1S_NYET);
2898 xudc_writel(xudc, val, PORTPM);
2899 }
2900
2901 val = xudc_readl(xudc, ST);
2902 if (val & ST_RC)
2903 xudc_writel(xudc, ST_RC, ST);
2904}
2905
2906static void tegra_xudc_port_disconnect(struct tegra_xudc *xudc)
2907{
2908 tegra_xudc_reset(xudc);
2909
2910 if (xudc->driver && xudc->driver->disconnect) {
2911 spin_unlock(&xudc->lock);
2912 xudc->driver->disconnect(&xudc->gadget);
2913 spin_lock(&xudc->lock);
2914 }
2915
2916 xudc->device_state = USB_STATE_NOTATTACHED;
2917 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2918
2919 complete(&xudc->disconnect_complete);
2920}
2921
2922static void tegra_xudc_port_reset(struct tegra_xudc *xudc)
2923{
2924 tegra_xudc_reset(xudc);
2925
2926 if (xudc->driver) {
2927 spin_unlock(&xudc->lock);
2928 usb_gadget_udc_reset(&xudc->gadget, xudc->driver);
2929 spin_lock(&xudc->lock);
2930 }
2931
2932 tegra_xudc_port_connect(xudc);
2933}
2934
2935static void tegra_xudc_port_suspend(struct tegra_xudc *xudc)
2936{
2937 dev_dbg(xudc->dev, "port suspend\n");
2938
2939 xudc->resume_state = xudc->device_state;
2940 xudc->device_state = USB_STATE_SUSPENDED;
2941 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2942
2943 if (xudc->driver->suspend) {
2944 spin_unlock(&xudc->lock);
2945 xudc->driver->suspend(&xudc->gadget);
2946 spin_lock(&xudc->lock);
2947 }
2948}
2949
2950static void tegra_xudc_port_resume(struct tegra_xudc *xudc)
2951{
2952 dev_dbg(xudc->dev, "port resume\n");
2953
2954 tegra_xudc_resume_device_state(xudc);
2955
2956 if (xudc->driver->resume) {
2957 spin_unlock(&xudc->lock);
2958 xudc->driver->resume(&xudc->gadget);
2959 spin_lock(&xudc->lock);
2960 }
2961}
2962
2963static inline void clear_port_change(struct tegra_xudc *xudc, u32 flag)
2964{
2965 u32 val;
2966
2967 val = xudc_readl(xudc, PORTSC);
2968 val &= ~PORTSC_CHANGE_MASK;
2969 val |= flag;
2970 xudc_writel(xudc, val, PORTSC);
2971}
2972
2973static void __tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
2974{
2975 u32 portsc, porthalt;
2976
2977 porthalt = xudc_readl(xudc, PORTHALT);
2978 if ((porthalt & PORTHALT_STCHG_REQ) &&
2979 (porthalt & PORTHALT_HALT_LTSSM)) {
2980 dev_dbg(xudc->dev, "STCHG_REQ, PORTHALT = %#x\n", porthalt);
2981 porthalt &= ~PORTHALT_HALT_LTSSM;
2982 xudc_writel(xudc, porthalt, PORTHALT);
2983 }
2984
2985 portsc = xudc_readl(xudc, PORTSC);
2986 if ((portsc & PORTSC_PRC) && (portsc & PORTSC_PR)) {
2987 dev_dbg(xudc->dev, "PRC, PR, PORTSC = %#x\n", portsc);
2988 clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
2989#define TOGGLE_VBUS_WAIT_MS 100
2990 if (xudc->soc->port_reset_quirk) {
2991 schedule_delayed_work(&xudc->port_reset_war_work,
2992 msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
2993 xudc->wait_for_sec_prc = 1;
2994 }
2995 }
2996
2997 if ((portsc & PORTSC_PRC) && !(portsc & PORTSC_PR)) {
2998 dev_dbg(xudc->dev, "PRC, Not PR, PORTSC = %#x\n", portsc);
2999 clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
3000 tegra_xudc_port_reset(xudc);
3001 cancel_delayed_work(&xudc->port_reset_war_work);
3002 xudc->wait_for_sec_prc = 0;
3003 }
3004
3005 portsc = xudc_readl(xudc, PORTSC);
3006 if (portsc & PORTSC_WRC) {
3007 dev_dbg(xudc->dev, "WRC, PORTSC = %#x\n", portsc);
3008 clear_port_change(xudc, PORTSC_WRC | PORTSC_PED);
3009 if (!(xudc_readl(xudc, PORTSC) & PORTSC_WPR))
3010 tegra_xudc_port_reset(xudc);
3011 }
3012
3013 portsc = xudc_readl(xudc, PORTSC);
3014 if (portsc & PORTSC_CSC) {
3015 dev_dbg(xudc->dev, "CSC, PORTSC = %#x\n", portsc);
3016 clear_port_change(xudc, PORTSC_CSC);
3017
3018 if (portsc & PORTSC_CCS)
3019 tegra_xudc_port_connect(xudc);
3020 else
3021 tegra_xudc_port_disconnect(xudc);
3022
3023 if (xudc->wait_csc) {
3024 cancel_delayed_work(&xudc->plc_reset_work);
3025 xudc->wait_csc = false;
3026 }
3027 }
3028
3029 portsc = xudc_readl(xudc, PORTSC);
3030 if (portsc & PORTSC_PLC) {
3031 u32 pls = (portsc & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT;
3032
3033 dev_dbg(xudc->dev, "PLC, PORTSC = %#x\n", portsc);
3034 clear_port_change(xudc, PORTSC_PLC);
3035 switch (pls) {
3036 case PORTSC_PLS_U3:
3037 tegra_xudc_port_suspend(xudc);
3038 break;
3039 case PORTSC_PLS_U0:
3040 if (xudc->gadget.speed < USB_SPEED_SUPER)
3041 tegra_xudc_port_resume(xudc);
3042 break;
3043 case PORTSC_PLS_RESUME:
3044 if (xudc->gadget.speed == USB_SPEED_SUPER)
3045 tegra_xudc_port_resume(xudc);
3046 break;
3047 case PORTSC_PLS_INACTIVE:
3048 schedule_delayed_work(&xudc->plc_reset_work,
3049 msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
3050 xudc->wait_csc = true;
3051 break;
3052 default:
3053 break;
3054 }
3055 }
3056
3057 if (portsc & PORTSC_CEC) {
3058 dev_warn(xudc->dev, "CEC, PORTSC = %#x\n", portsc);
3059 clear_port_change(xudc, PORTSC_CEC);
3060 }
3061
3062 dev_dbg(xudc->dev, "PORTSC = %#x\n", xudc_readl(xudc, PORTSC));
3063}
3064
3065static void tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
3066{
3067 while ((xudc_readl(xudc, PORTSC) & PORTSC_CHANGE_MASK) ||
3068 (xudc_readl(xudc, PORTHALT) & PORTHALT_STCHG_REQ))
3069 __tegra_xudc_handle_port_status(xudc);
3070}
3071
3072static void tegra_xudc_handle_event(struct tegra_xudc *xudc,
3073 struct tegra_xudc_trb *event)
3074{
3075 u32 type = trb_read_type(event);
3076
3077 dump_trb(xudc, "EVENT", event);
3078
3079 switch (type) {
3080 case TRB_TYPE_PORT_STATUS_CHANGE_EVENT:
3081 tegra_xudc_handle_port_status(xudc);
3082 break;
3083 case TRB_TYPE_TRANSFER_EVENT:
3084 tegra_xudc_handle_transfer_event(xudc, event);
3085 break;
3086 case TRB_TYPE_SETUP_PACKET_EVENT:
3087 tegra_xudc_handle_ep0_event(xudc, event);
3088 break;
3089 default:
3090 dev_info(xudc->dev, "Unrecognized TRB type = %#x\n", type);
3091 break;
3092 }
3093}
3094
3095static void tegra_xudc_process_event_ring(struct tegra_xudc *xudc)
3096{
3097 struct tegra_xudc_trb *event;
3098 dma_addr_t erdp;
3099
3100 while (true) {
3101 event = xudc->event_ring[xudc->event_ring_index] +
3102 xudc->event_ring_deq_ptr;
3103
3104 if (trb_read_cycle(event) != xudc->ccs)
3105 break;
3106
3107 tegra_xudc_handle_event(xudc, event);
3108
3109 xudc->event_ring_deq_ptr++;
3110 if (xudc->event_ring_deq_ptr == XUDC_EVENT_RING_SIZE) {
3111 xudc->event_ring_deq_ptr = 0;
3112 xudc->event_ring_index++;
3113 }
3114
3115 if (xudc->event_ring_index == XUDC_NR_EVENT_RINGS) {
3116 xudc->event_ring_index = 0;
3117 xudc->ccs = !xudc->ccs;
3118 }
3119 }
3120
3121 erdp = xudc->event_ring_phys[xudc->event_ring_index] +
3122 xudc->event_ring_deq_ptr * sizeof(*event);
3123
3124 xudc_writel(xudc, upper_32_bits(erdp), ERDPHI);
3125 xudc_writel(xudc, lower_32_bits(erdp) | ERDPLO_EHB, ERDPLO);
3126}
3127
3128static irqreturn_t tegra_xudc_irq(int irq, void *data)
3129{
3130 struct tegra_xudc *xudc = data;
3131 unsigned long flags;
3132 u32 val;
3133
3134 val = xudc_readl(xudc, ST);
3135 if (!(val & ST_IP))
3136 return IRQ_NONE;
3137 xudc_writel(xudc, ST_IP, ST);
3138
3139 spin_lock_irqsave(&xudc->lock, flags);
3140 tegra_xudc_process_event_ring(xudc);
3141 spin_unlock_irqrestore(&xudc->lock, flags);
3142
3143 return IRQ_HANDLED;
3144}
3145
3146static int tegra_xudc_alloc_ep(struct tegra_xudc *xudc, unsigned int index)
3147{
3148 struct tegra_xudc_ep *ep = &xudc->ep[index];
3149
3150 ep->xudc = xudc;
3151 ep->index = index;
3152 ep->context = &xudc->ep_context[index];
3153 INIT_LIST_HEAD(&ep->queue);
3154
3155 /*
3156 * EP1 would be the input endpoint corresponding to EP0, but since
3157 * EP0 is bi-directional, EP1 is unused.
3158 */
3159 if (index == 1)
3160 return 0;
3161
3162 ep->transfer_ring = dma_pool_alloc(xudc->transfer_ring_pool,
3163 GFP_KERNEL,
3164 &ep->transfer_ring_phys);
3165 if (!ep->transfer_ring)
3166 return -ENOMEM;
3167
3168 if (index) {
3169 snprintf(ep->name, sizeof(ep->name), "ep%u%s", index / 2,
3170 (index % 2 == 0) ? "out" : "in");
3171 ep->usb_ep.name = ep->name;
3172 usb_ep_set_maxpacket_limit(&ep->usb_ep, 1024);
3173 ep->usb_ep.max_streams = 16;
3174 ep->usb_ep.ops = &tegra_xudc_ep_ops;
3175 ep->usb_ep.caps.type_bulk = true;
3176 ep->usb_ep.caps.type_int = true;
3177 if (index & 1)
3178 ep->usb_ep.caps.dir_in = true;
3179 else
3180 ep->usb_ep.caps.dir_out = true;
3181 list_add_tail(&ep->usb_ep.ep_list, &xudc->gadget.ep_list);
3182 } else {
3183 strscpy(ep->name, "ep0", 3);
3184 ep->usb_ep.name = ep->name;
3185 usb_ep_set_maxpacket_limit(&ep->usb_ep, 512);
3186 ep->usb_ep.ops = &tegra_xudc_ep0_ops;
3187 ep->usb_ep.caps.type_control = true;
3188 ep->usb_ep.caps.dir_in = true;
3189 ep->usb_ep.caps.dir_out = true;
3190 }
3191
3192 return 0;
3193}
3194
3195static void tegra_xudc_free_ep(struct tegra_xudc *xudc, unsigned int index)
3196{
3197 struct tegra_xudc_ep *ep = &xudc->ep[index];
3198
3199 /*
3200 * EP1 would be the input endpoint corresponding to EP0, but since
3201 * EP0 is bi-directional, EP1 is unused.
3202 */
3203 if (index == 1)
3204 return;
3205
3206 dma_pool_free(xudc->transfer_ring_pool, ep->transfer_ring,
3207 ep->transfer_ring_phys);
3208}
3209
3210static int tegra_xudc_alloc_eps(struct tegra_xudc *xudc)
3211{
3212 struct usb_request *req;
3213 unsigned int i;
3214 int err;
3215
3216 xudc->ep_context =
3217 dma_alloc_coherent(xudc->dev, XUDC_NR_EPS *
3218 sizeof(*xudc->ep_context),
3219 &xudc->ep_context_phys, GFP_KERNEL);
3220 if (!xudc->ep_context)
3221 return -ENOMEM;
3222
3223 xudc->transfer_ring_pool =
3224 dmam_pool_create(dev_name(xudc->dev), xudc->dev,
3225 XUDC_TRANSFER_RING_SIZE *
3226 sizeof(struct tegra_xudc_trb),
3227 sizeof(struct tegra_xudc_trb), 0);
3228 if (!xudc->transfer_ring_pool) {
3229 err = -ENOMEM;
3230 goto free_ep_context;
3231 }
3232
3233 INIT_LIST_HEAD(&xudc->gadget.ep_list);
3234 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
3235 err = tegra_xudc_alloc_ep(xudc, i);
3236 if (err < 0)
3237 goto free_eps;
3238 }
3239
3240 req = tegra_xudc_ep_alloc_request(&xudc->ep[0].usb_ep, GFP_KERNEL);
3241 if (!req) {
3242 err = -ENOMEM;
3243 goto free_eps;
3244 }
3245 xudc->ep0_req = to_xudc_req(req);
3246
3247 return 0;
3248
3249free_eps:
3250 for (; i > 0; i--)
3251 tegra_xudc_free_ep(xudc, i - 1);
3252free_ep_context:
3253 dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3254 xudc->ep_context, xudc->ep_context_phys);
3255 return err;
3256}
3257
3258static void tegra_xudc_init_eps(struct tegra_xudc *xudc)
3259{
3260 xudc_writel(xudc, lower_32_bits(xudc->ep_context_phys), ECPLO);
3261 xudc_writel(xudc, upper_32_bits(xudc->ep_context_phys), ECPHI);
3262}
3263
3264static void tegra_xudc_free_eps(struct tegra_xudc *xudc)
3265{
3266 unsigned int i;
3267
3268 tegra_xudc_ep_free_request(&xudc->ep[0].usb_ep,
3269 &xudc->ep0_req->usb_req);
3270
3271 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
3272 tegra_xudc_free_ep(xudc, i);
3273
3274 dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3275 xudc->ep_context, xudc->ep_context_phys);
3276}
3277
3278static int tegra_xudc_alloc_event_ring(struct tegra_xudc *xudc)
3279{
3280 unsigned int i;
3281
3282 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3283 xudc->event_ring[i] =
3284 dma_alloc_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3285 sizeof(*xudc->event_ring[i]),
3286 &xudc->event_ring_phys[i],
3287 GFP_KERNEL);
3288 if (!xudc->event_ring[i])
3289 goto free_dma;
3290 }
3291
3292 return 0;
3293
3294free_dma:
3295 for (; i > 0; i--) {
3296 dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3297 sizeof(*xudc->event_ring[i - 1]),
3298 xudc->event_ring[i - 1],
3299 xudc->event_ring_phys[i - 1]);
3300 }
3301 return -ENOMEM;
3302}
3303
3304static void tegra_xudc_init_event_ring(struct tegra_xudc *xudc)
3305{
3306 unsigned int i;
3307 u32 val;
3308
3309 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3310 memset(xudc->event_ring[i], 0, XUDC_EVENT_RING_SIZE *
3311 sizeof(*xudc->event_ring[i]));
3312
3313 val = xudc_readl(xudc, ERSTSZ);
3314 val &= ~(ERSTSZ_ERSTXSZ_MASK << ERSTSZ_ERSTXSZ_SHIFT(i));
3315 val |= XUDC_EVENT_RING_SIZE << ERSTSZ_ERSTXSZ_SHIFT(i);
3316 xudc_writel(xudc, val, ERSTSZ);
3317
3318 xudc_writel(xudc, lower_32_bits(xudc->event_ring_phys[i]),
3319 ERSTXBALO(i));
3320 xudc_writel(xudc, upper_32_bits(xudc->event_ring_phys[i]),
3321 ERSTXBAHI(i));
3322 }
3323
3324 val = lower_32_bits(xudc->event_ring_phys[0]);
3325 xudc_writel(xudc, val, ERDPLO);
3326 val |= EREPLO_ECS;
3327 xudc_writel(xudc, val, EREPLO);
3328
3329 val = upper_32_bits(xudc->event_ring_phys[0]);
3330 xudc_writel(xudc, val, ERDPHI);
3331 xudc_writel(xudc, val, EREPHI);
3332
3333 xudc->ccs = true;
3334 xudc->event_ring_index = 0;
3335 xudc->event_ring_deq_ptr = 0;
3336}
3337
3338static void tegra_xudc_free_event_ring(struct tegra_xudc *xudc)
3339{
3340 unsigned int i;
3341
3342 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3343 dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3344 sizeof(*xudc->event_ring[i]),
3345 xudc->event_ring[i],
3346 xudc->event_ring_phys[i]);
3347 }
3348}
3349
3350static void tegra_xudc_fpci_ipfs_init(struct tegra_xudc *xudc)
3351{
3352 u32 val;
3353
3354 if (xudc->soc->has_ipfs) {
3355 val = ipfs_readl(xudc, XUSB_DEV_CONFIGURATION_0);
3356 val |= XUSB_DEV_CONFIGURATION_0_EN_FPCI;
3357 ipfs_writel(xudc, val, XUSB_DEV_CONFIGURATION_0);
3358 usleep_range(10, 15);
3359 }
3360
3361 /* Enable bus master */
3362 val = XUSB_DEV_CFG_1_IO_SPACE_EN | XUSB_DEV_CFG_1_MEMORY_SPACE_EN |
3363 XUSB_DEV_CFG_1_BUS_MASTER_EN;
3364 fpci_writel(xudc, val, XUSB_DEV_CFG_1);
3365
3366 /* Program BAR0 space */
3367 val = fpci_readl(xudc, XUSB_DEV_CFG_4);
3368 val &= ~(XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3369 val |= xudc->phys_base & (XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3370
3371 fpci_writel(xudc, val, XUSB_DEV_CFG_4);
3372 fpci_writel(xudc, upper_32_bits(xudc->phys_base), XUSB_DEV_CFG_5);
3373
3374 usleep_range(100, 200);
3375
3376 if (xudc->soc->has_ipfs) {
3377 /* Enable interrupt assertion */
3378 val = ipfs_readl(xudc, XUSB_DEV_INTR_MASK_0);
3379 val |= XUSB_DEV_INTR_MASK_0_IP_INT_MASK;
3380 ipfs_writel(xudc, val, XUSB_DEV_INTR_MASK_0);
3381 }
3382}
3383
3384static void tegra_xudc_device_params_init(struct tegra_xudc *xudc)
3385{
3386 u32 val, imod;
3387
3388 if (xudc->soc->has_ipfs) {
3389 val = xudc_readl(xudc, BLCG);
3390 val |= BLCG_ALL;
3391 val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE |
3392 BLCG_COREPLL_PWRDN);
3393 val |= BLCG_IOPLL_0_PWRDN;
3394 val |= BLCG_IOPLL_1_PWRDN;
3395 val |= BLCG_IOPLL_2_PWRDN;
3396
3397 xudc_writel(xudc, val, BLCG);
3398 }
3399
3400 if (xudc->soc->port_speed_quirk)
3401 tegra_xudc_limit_port_speed(xudc);
3402
3403 /* Set a reasonable U3 exit timer value. */
3404 val = xudc_readl(xudc, SSPX_CORE_PADCTL4);
3405 val &= ~(SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK);
3406 val |= SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(0x5dc0);
3407 xudc_writel(xudc, val, SSPX_CORE_PADCTL4);
3408
3409 /* Default ping LFPS tBurst is too large. */
3410 val = xudc_readl(xudc, SSPX_CORE_CNT0);
3411 val &= ~(SSPX_CORE_CNT0_PING_TBURST_MASK);
3412 val |= SSPX_CORE_CNT0_PING_TBURST(0xa);
3413 xudc_writel(xudc, val, SSPX_CORE_CNT0);
3414
3415 /* Default tPortConfiguration timeout is too small. */
3416 val = xudc_readl(xudc, SSPX_CORE_CNT30);
3417 val &= ~(SSPX_CORE_CNT30_LMPITP_TIMER_MASK);
3418 val |= SSPX_CORE_CNT30_LMPITP_TIMER(0x978);
3419 xudc_writel(xudc, val, SSPX_CORE_CNT30);
3420
3421 if (xudc->soc->lpm_enable) {
3422 /* Set L1 resume duration to 95 us. */
3423 val = xudc_readl(xudc, HSFSPI_COUNT13);
3424 val &= ~(HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK);
3425 val |= HSFSPI_COUNT13_U2_RESUME_K_DURATION(0x2c88);
3426 xudc_writel(xudc, val, HSFSPI_COUNT13);
3427 }
3428
3429 /*
3430 * Compliance suite appears to be violating polling LFPS tBurst max
3431 * of 1.4us. Send 1.45us instead.
3432 */
3433 val = xudc_readl(xudc, SSPX_CORE_CNT32);
3434 val &= ~(SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK);
3435 val |= SSPX_CORE_CNT32_POLL_TBURST_MAX(0xb0);
3436 xudc_writel(xudc, val, SSPX_CORE_CNT32);
3437
3438 /* Direct HS/FS port instance to RxDetect. */
3439 val = xudc_readl(xudc, CFG_DEV_FE);
3440 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3441 val |= CFG_DEV_FE_PORTREGSEL(CFG_DEV_FE_PORTREGSEL_HSFS_PI);
3442 xudc_writel(xudc, val, CFG_DEV_FE);
3443
3444 val = xudc_readl(xudc, PORTSC);
3445 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3446 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3447 xudc_writel(xudc, val, PORTSC);
3448
3449 /* Direct SS port instance to RxDetect. */
3450 val = xudc_readl(xudc, CFG_DEV_FE);
3451 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3452 val |= CFG_DEV_FE_PORTREGSEL_SS_PI & CFG_DEV_FE_PORTREGSEL_MASK;
3453 xudc_writel(xudc, val, CFG_DEV_FE);
3454
3455 val = xudc_readl(xudc, PORTSC);
3456 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3457 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3458 xudc_writel(xudc, val, PORTSC);
3459
3460 /* Restore port instance. */
3461 val = xudc_readl(xudc, CFG_DEV_FE);
3462 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3463 xudc_writel(xudc, val, CFG_DEV_FE);
3464
3465 /*
3466 * Enable INFINITE_SS_RETRY to prevent device from entering
3467 * Disabled.Error when attached to buggy SuperSpeed hubs.
3468 */
3469 val = xudc_readl(xudc, CFG_DEV_FE);
3470 val |= CFG_DEV_FE_INFINITE_SS_RETRY;
3471 xudc_writel(xudc, val, CFG_DEV_FE);
3472
3473 /* Set interrupt moderation. */
3474 imod = XUDC_INTERRUPT_MODERATION_US * 4;
3475 val = xudc_readl(xudc, RT_IMOD);
3476 val &= ~((RT_IMOD_IMODI_MASK) | (RT_IMOD_IMODC_MASK));
3477 val |= (RT_IMOD_IMODI(imod) | RT_IMOD_IMODC(imod));
3478 xudc_writel(xudc, val, RT_IMOD);
3479
3480 /* increase SSPI transaction timeout from 32us to 512us */
3481 val = xudc_readl(xudc, CFG_DEV_SSPI_XFER);
3482 val &= ~(CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK);
3483 val |= CFG_DEV_SSPI_XFER_ACKTIMEOUT(0xf000);
3484 xudc_writel(xudc, val, CFG_DEV_SSPI_XFER);
3485}
3486
3487static int tegra_xudc_phy_get(struct tegra_xudc *xudc)
3488{
3489 int err = 0, usb3;
3490 unsigned int i;
3491
3492 xudc->utmi_phy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3493 sizeof(*xudc->utmi_phy), GFP_KERNEL);
3494 if (!xudc->utmi_phy)
3495 return -ENOMEM;
3496
3497 xudc->usb3_phy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3498 sizeof(*xudc->usb3_phy), GFP_KERNEL);
3499 if (!xudc->usb3_phy)
3500 return -ENOMEM;
3501
3502 xudc->usbphy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3503 sizeof(*xudc->usbphy), GFP_KERNEL);
3504 if (!xudc->usbphy)
3505 return -ENOMEM;
3506
3507 xudc->vbus_nb.notifier_call = tegra_xudc_vbus_notify;
3508
3509 for (i = 0; i < xudc->soc->num_phys; i++) {
3510 char phy_name[] = "usb.-.";
3511
3512 /* Get USB2 phy */
3513 snprintf(phy_name, sizeof(phy_name), "usb2-%d", i);
3514 xudc->utmi_phy[i] = devm_phy_optional_get(xudc->dev, phy_name);
3515 if (IS_ERR(xudc->utmi_phy[i])) {
3516 err = PTR_ERR(xudc->utmi_phy[i]);
3517 dev_err_probe(xudc->dev, err,
3518 "failed to get usb2-%d PHY\n", i);
3519 goto clean_up;
3520 } else if (xudc->utmi_phy[i]) {
3521 /* Get usb-phy, if utmi phy is available */
3522 xudc->usbphy[i] = devm_usb_get_phy_by_node(xudc->dev,
3523 xudc->utmi_phy[i]->dev.of_node,
3524 &xudc->vbus_nb);
3525 if (IS_ERR(xudc->usbphy[i])) {
3526 err = PTR_ERR(xudc->usbphy[i]);
3527 dev_err_probe(xudc->dev, err,
3528 "failed to get usbphy-%d\n", i);
3529 goto clean_up;
3530 }
3531 } else if (!xudc->utmi_phy[i]) {
3532 /* if utmi phy is not available, ignore USB3 phy get */
3533 continue;
3534 }
3535
3536 /* Get USB3 phy */
3537 usb3 = tegra_xusb_padctl_get_usb3_companion(xudc->padctl, i);
3538 if (usb3 < 0)
3539 continue;
3540
3541 snprintf(phy_name, sizeof(phy_name), "usb3-%d", usb3);
3542 xudc->usb3_phy[i] = devm_phy_optional_get(xudc->dev, phy_name);
3543 if (IS_ERR(xudc->usb3_phy[i])) {
3544 err = PTR_ERR(xudc->usb3_phy[i]);
3545 dev_err_probe(xudc->dev, err,
3546 "failed to get usb3-%d PHY\n", usb3);
3547 goto clean_up;
3548 } else if (xudc->usb3_phy[i])
3549 dev_dbg(xudc->dev, "usb3-%d PHY registered", usb3);
3550 }
3551
3552 return err;
3553
3554clean_up:
3555 for (i = 0; i < xudc->soc->num_phys; i++) {
3556 xudc->usb3_phy[i] = NULL;
3557 xudc->utmi_phy[i] = NULL;
3558 xudc->usbphy[i] = NULL;
3559 }
3560
3561 return err;
3562}
3563
3564static void tegra_xudc_phy_exit(struct tegra_xudc *xudc)
3565{
3566 unsigned int i;
3567
3568 for (i = 0; i < xudc->soc->num_phys; i++) {
3569 phy_exit(xudc->usb3_phy[i]);
3570 phy_exit(xudc->utmi_phy[i]);
3571 }
3572}
3573
3574static int tegra_xudc_phy_init(struct tegra_xudc *xudc)
3575{
3576 int err;
3577 unsigned int i;
3578
3579 for (i = 0; i < xudc->soc->num_phys; i++) {
3580 err = phy_init(xudc->utmi_phy[i]);
3581 if (err < 0) {
3582 dev_err(xudc->dev, "UTMI PHY #%u initialization failed: %d\n", i, err);
3583 goto exit_phy;
3584 }
3585
3586 err = phy_init(xudc->usb3_phy[i]);
3587 if (err < 0) {
3588 dev_err(xudc->dev, "USB3 PHY #%u initialization failed: %d\n", i, err);
3589 goto exit_phy;
3590 }
3591 }
3592 return 0;
3593
3594exit_phy:
3595 tegra_xudc_phy_exit(xudc);
3596 return err;
3597}
3598
3599static const char * const tegra210_xudc_supply_names[] = {
3600 "hvdd-usb",
3601 "avddio-usb",
3602};
3603
3604static const char * const tegra210_xudc_clock_names[] = {
3605 "dev",
3606 "ss",
3607 "ss_src",
3608 "hs_src",
3609 "fs_src",
3610};
3611
3612static const char * const tegra186_xudc_clock_names[] = {
3613 "dev",
3614 "ss",
3615 "ss_src",
3616 "fs_src",
3617};
3618
3619static struct tegra_xudc_soc tegra210_xudc_soc_data = {
3620 .supply_names = tegra210_xudc_supply_names,
3621 .num_supplies = ARRAY_SIZE(tegra210_xudc_supply_names),
3622 .clock_names = tegra210_xudc_clock_names,
3623 .num_clks = ARRAY_SIZE(tegra210_xudc_clock_names),
3624 .num_phys = 4,
3625 .u1_enable = false,
3626 .u2_enable = true,
3627 .lpm_enable = false,
3628 .invalid_seq_num = true,
3629 .pls_quirk = true,
3630 .port_reset_quirk = true,
3631 .port_speed_quirk = false,
3632 .has_ipfs = true,
3633};
3634
3635static struct tegra_xudc_soc tegra186_xudc_soc_data = {
3636 .clock_names = tegra186_xudc_clock_names,
3637 .num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3638 .num_phys = 4,
3639 .u1_enable = true,
3640 .u2_enable = true,
3641 .lpm_enable = false,
3642 .invalid_seq_num = false,
3643 .pls_quirk = false,
3644 .port_reset_quirk = false,
3645 .port_speed_quirk = false,
3646 .has_ipfs = false,
3647};
3648
3649static struct tegra_xudc_soc tegra194_xudc_soc_data = {
3650 .clock_names = tegra186_xudc_clock_names,
3651 .num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3652 .num_phys = 4,
3653 .u1_enable = true,
3654 .u2_enable = true,
3655 .lpm_enable = true,
3656 .invalid_seq_num = false,
3657 .pls_quirk = false,
3658 .port_reset_quirk = false,
3659 .port_speed_quirk = true,
3660 .has_ipfs = false,
3661};
3662
3663static const struct of_device_id tegra_xudc_of_match[] = {
3664 {
3665 .compatible = "nvidia,tegra210-xudc",
3666 .data = &tegra210_xudc_soc_data
3667 },
3668 {
3669 .compatible = "nvidia,tegra186-xudc",
3670 .data = &tegra186_xudc_soc_data
3671 },
3672 {
3673 .compatible = "nvidia,tegra194-xudc",
3674 .data = &tegra194_xudc_soc_data
3675 },
3676 { }
3677};
3678MODULE_DEVICE_TABLE(of, tegra_xudc_of_match);
3679
3680static void tegra_xudc_powerdomain_remove(struct tegra_xudc *xudc)
3681{
3682 if (xudc->genpd_dl_ss)
3683 device_link_del(xudc->genpd_dl_ss);
3684 if (xudc->genpd_dl_device)
3685 device_link_del(xudc->genpd_dl_device);
3686 if (xudc->genpd_dev_ss)
3687 dev_pm_domain_detach(xudc->genpd_dev_ss, true);
3688 if (xudc->genpd_dev_device)
3689 dev_pm_domain_detach(xudc->genpd_dev_device, true);
3690}
3691
3692static int tegra_xudc_powerdomain_init(struct tegra_xudc *xudc)
3693{
3694 struct device *dev = xudc->dev;
3695 int err;
3696
3697 xudc->genpd_dev_device = dev_pm_domain_attach_by_name(dev, "dev");
3698 if (IS_ERR_OR_NULL(xudc->genpd_dev_device)) {
3699 err = PTR_ERR(xudc->genpd_dev_device) ? : -ENODATA;
3700 dev_err(dev, "failed to get device power domain: %d\n", err);
3701 return err;
3702 }
3703
3704 xudc->genpd_dev_ss = dev_pm_domain_attach_by_name(dev, "ss");
3705 if (IS_ERR_OR_NULL(xudc->genpd_dev_ss)) {
3706 err = PTR_ERR(xudc->genpd_dev_ss) ? : -ENODATA;
3707 dev_err(dev, "failed to get SuperSpeed power domain: %d\n", err);
3708 return err;
3709 }
3710
3711 xudc->genpd_dl_device = device_link_add(dev, xudc->genpd_dev_device,
3712 DL_FLAG_PM_RUNTIME |
3713 DL_FLAG_STATELESS);
3714 if (!xudc->genpd_dl_device) {
3715 dev_err(dev, "failed to add USB device link\n");
3716 return -ENODEV;
3717 }
3718
3719 xudc->genpd_dl_ss = device_link_add(dev, xudc->genpd_dev_ss,
3720 DL_FLAG_PM_RUNTIME |
3721 DL_FLAG_STATELESS);
3722 if (!xudc->genpd_dl_ss) {
3723 dev_err(dev, "failed to add SuperSpeed device link\n");
3724 return -ENODEV;
3725 }
3726
3727 return 0;
3728}
3729
3730static int tegra_xudc_probe(struct platform_device *pdev)
3731{
3732 struct tegra_xudc *xudc;
3733 struct resource *res;
3734 unsigned int i;
3735 int err;
3736
3737 xudc = devm_kzalloc(&pdev->dev, sizeof(*xudc), GFP_KERNEL);
3738 if (!xudc)
3739 return -ENOMEM;
3740
3741 xudc->dev = &pdev->dev;
3742 platform_set_drvdata(pdev, xudc);
3743
3744 xudc->soc = of_device_get_match_data(&pdev->dev);
3745 if (!xudc->soc)
3746 return -ENODEV;
3747
3748 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3749 xudc->base = devm_ioremap_resource(&pdev->dev, res);
3750 if (IS_ERR(xudc->base))
3751 return PTR_ERR(xudc->base);
3752 xudc->phys_base = res->start;
3753
3754 xudc->fpci = devm_platform_ioremap_resource_byname(pdev, "fpci");
3755 if (IS_ERR(xudc->fpci))
3756 return PTR_ERR(xudc->fpci);
3757
3758 if (xudc->soc->has_ipfs) {
3759 xudc->ipfs = devm_platform_ioremap_resource_byname(pdev, "ipfs");
3760 if (IS_ERR(xudc->ipfs))
3761 return PTR_ERR(xudc->ipfs);
3762 }
3763
3764 xudc->irq = platform_get_irq(pdev, 0);
3765 if (xudc->irq < 0)
3766 return xudc->irq;
3767
3768 err = devm_request_irq(&pdev->dev, xudc->irq, tegra_xudc_irq, 0,
3769 dev_name(&pdev->dev), xudc);
3770 if (err < 0) {
3771 dev_err(xudc->dev, "failed to claim IRQ#%u: %d\n", xudc->irq,
3772 err);
3773 return err;
3774 }
3775
3776 xudc->clks = devm_kcalloc(&pdev->dev, xudc->soc->num_clks, sizeof(*xudc->clks),
3777 GFP_KERNEL);
3778 if (!xudc->clks)
3779 return -ENOMEM;
3780
3781 for (i = 0; i < xudc->soc->num_clks; i++)
3782 xudc->clks[i].id = xudc->soc->clock_names[i];
3783
3784 err = devm_clk_bulk_get(&pdev->dev, xudc->soc->num_clks, xudc->clks);
3785 if (err) {
3786 dev_err_probe(xudc->dev, err, "failed to request clocks\n");
3787 return err;
3788 }
3789
3790 xudc->supplies = devm_kcalloc(&pdev->dev, xudc->soc->num_supplies,
3791 sizeof(*xudc->supplies), GFP_KERNEL);
3792 if (!xudc->supplies)
3793 return -ENOMEM;
3794
3795 for (i = 0; i < xudc->soc->num_supplies; i++)
3796 xudc->supplies[i].supply = xudc->soc->supply_names[i];
3797
3798 err = devm_regulator_bulk_get(&pdev->dev, xudc->soc->num_supplies,
3799 xudc->supplies);
3800 if (err) {
3801 dev_err_probe(xudc->dev, err, "failed to request regulators\n");
3802 return err;
3803 }
3804
3805 xudc->padctl = tegra_xusb_padctl_get(&pdev->dev);
3806 if (IS_ERR(xudc->padctl))
3807 return PTR_ERR(xudc->padctl);
3808
3809 err = regulator_bulk_enable(xudc->soc->num_supplies, xudc->supplies);
3810 if (err) {
3811 dev_err(xudc->dev, "failed to enable regulators: %d\n", err);
3812 goto put_padctl;
3813 }
3814
3815 err = tegra_xudc_phy_get(xudc);
3816 if (err)
3817 goto disable_regulator;
3818
3819 err = tegra_xudc_powerdomain_init(xudc);
3820 if (err)
3821 goto put_powerdomains;
3822
3823 err = tegra_xudc_phy_init(xudc);
3824 if (err)
3825 goto put_powerdomains;
3826
3827 err = tegra_xudc_alloc_event_ring(xudc);
3828 if (err)
3829 goto disable_phy;
3830
3831 err = tegra_xudc_alloc_eps(xudc);
3832 if (err)
3833 goto free_event_ring;
3834
3835 spin_lock_init(&xudc->lock);
3836
3837 init_completion(&xudc->disconnect_complete);
3838
3839 INIT_WORK(&xudc->usb_role_sw_work, tegra_xudc_usb_role_sw_work);
3840
3841 INIT_DELAYED_WORK(&xudc->plc_reset_work, tegra_xudc_plc_reset_work);
3842
3843 INIT_DELAYED_WORK(&xudc->port_reset_war_work,
3844 tegra_xudc_port_reset_war_work);
3845
3846 pm_runtime_enable(&pdev->dev);
3847
3848 xudc->gadget.ops = &tegra_xudc_gadget_ops;
3849 xudc->gadget.ep0 = &xudc->ep[0].usb_ep;
3850 xudc->gadget.name = "tegra-xudc";
3851 xudc->gadget.max_speed = USB_SPEED_SUPER;
3852
3853 err = usb_add_gadget_udc(&pdev->dev, &xudc->gadget);
3854 if (err) {
3855 dev_err(&pdev->dev, "failed to add USB gadget: %d\n", err);
3856 goto free_eps;
3857 }
3858
3859 return 0;
3860
3861free_eps:
3862 pm_runtime_disable(&pdev->dev);
3863 tegra_xudc_free_eps(xudc);
3864free_event_ring:
3865 tegra_xudc_free_event_ring(xudc);
3866disable_phy:
3867 tegra_xudc_phy_exit(xudc);
3868put_powerdomains:
3869 tegra_xudc_powerdomain_remove(xudc);
3870disable_regulator:
3871 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3872put_padctl:
3873 tegra_xusb_padctl_put(xudc->padctl);
3874
3875 return err;
3876}
3877
3878static int tegra_xudc_remove(struct platform_device *pdev)
3879{
3880 struct tegra_xudc *xudc = platform_get_drvdata(pdev);
3881 unsigned int i;
3882
3883 pm_runtime_get_sync(xudc->dev);
3884
3885 cancel_delayed_work_sync(&xudc->plc_reset_work);
3886 cancel_work_sync(&xudc->usb_role_sw_work);
3887
3888 usb_del_gadget_udc(&xudc->gadget);
3889
3890 tegra_xudc_free_eps(xudc);
3891 tegra_xudc_free_event_ring(xudc);
3892
3893 tegra_xudc_powerdomain_remove(xudc);
3894
3895 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3896
3897 for (i = 0; i < xudc->soc->num_phys; i++) {
3898 phy_power_off(xudc->utmi_phy[i]);
3899 phy_power_off(xudc->usb3_phy[i]);
3900 }
3901
3902 tegra_xudc_phy_exit(xudc);
3903
3904 pm_runtime_disable(xudc->dev);
3905 pm_runtime_put(xudc->dev);
3906
3907 tegra_xusb_padctl_put(xudc->padctl);
3908
3909 return 0;
3910}
3911
3912static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc)
3913{
3914 unsigned long flags;
3915
3916 dev_dbg(xudc->dev, "entering ELPG\n");
3917
3918 spin_lock_irqsave(&xudc->lock, flags);
3919
3920 xudc->powergated = true;
3921 xudc->saved_regs.ctrl = xudc_readl(xudc, CTRL);
3922 xudc->saved_regs.portpm = xudc_readl(xudc, PORTPM);
3923 xudc_writel(xudc, 0, CTRL);
3924
3925 spin_unlock_irqrestore(&xudc->lock, flags);
3926
3927 clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks);
3928
3929 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3930
3931 dev_dbg(xudc->dev, "entering ELPG done\n");
3932 return 0;
3933}
3934
3935static int __maybe_unused tegra_xudc_unpowergate(struct tegra_xudc *xudc)
3936{
3937 unsigned long flags;
3938 int err;
3939
3940 dev_dbg(xudc->dev, "exiting ELPG\n");
3941
3942 err = regulator_bulk_enable(xudc->soc->num_supplies,
3943 xudc->supplies);
3944 if (err < 0)
3945 return err;
3946
3947 err = clk_bulk_prepare_enable(xudc->soc->num_clks, xudc->clks);
3948 if (err < 0)
3949 return err;
3950
3951 tegra_xudc_fpci_ipfs_init(xudc);
3952
3953 tegra_xudc_device_params_init(xudc);
3954
3955 tegra_xudc_init_event_ring(xudc);
3956
3957 tegra_xudc_init_eps(xudc);
3958
3959 xudc_writel(xudc, xudc->saved_regs.portpm, PORTPM);
3960 xudc_writel(xudc, xudc->saved_regs.ctrl, CTRL);
3961
3962 spin_lock_irqsave(&xudc->lock, flags);
3963 xudc->powergated = false;
3964 spin_unlock_irqrestore(&xudc->lock, flags);
3965
3966 dev_dbg(xudc->dev, "exiting ELPG done\n");
3967 return 0;
3968}
3969
3970static int __maybe_unused tegra_xudc_suspend(struct device *dev)
3971{
3972 struct tegra_xudc *xudc = dev_get_drvdata(dev);
3973 unsigned long flags;
3974
3975 spin_lock_irqsave(&xudc->lock, flags);
3976 xudc->suspended = true;
3977 spin_unlock_irqrestore(&xudc->lock, flags);
3978
3979 flush_work(&xudc->usb_role_sw_work);
3980
3981 if (!pm_runtime_status_suspended(dev)) {
3982 /* Forcibly disconnect before powergating. */
3983 tegra_xudc_device_mode_off(xudc);
3984 tegra_xudc_powergate(xudc);
3985 }
3986
3987 pm_runtime_disable(dev);
3988
3989 return 0;
3990}
3991
3992static int __maybe_unused tegra_xudc_resume(struct device *dev)
3993{
3994 struct tegra_xudc *xudc = dev_get_drvdata(dev);
3995 unsigned long flags;
3996 int err;
3997
3998 err = tegra_xudc_unpowergate(xudc);
3999 if (err < 0)
4000 return err;
4001
4002 spin_lock_irqsave(&xudc->lock, flags);
4003 xudc->suspended = false;
4004 spin_unlock_irqrestore(&xudc->lock, flags);
4005
4006 schedule_work(&xudc->usb_role_sw_work);
4007
4008 pm_runtime_enable(dev);
4009
4010 return 0;
4011}
4012
4013static int __maybe_unused tegra_xudc_runtime_suspend(struct device *dev)
4014{
4015 struct tegra_xudc *xudc = dev_get_drvdata(dev);
4016
4017 return tegra_xudc_powergate(xudc);
4018}
4019
4020static int __maybe_unused tegra_xudc_runtime_resume(struct device *dev)
4021{
4022 struct tegra_xudc *xudc = dev_get_drvdata(dev);
4023
4024 return tegra_xudc_unpowergate(xudc);
4025}
4026
4027static const struct dev_pm_ops tegra_xudc_pm_ops = {
4028 SET_SYSTEM_SLEEP_PM_OPS(tegra_xudc_suspend, tegra_xudc_resume)
4029 SET_RUNTIME_PM_OPS(tegra_xudc_runtime_suspend,
4030 tegra_xudc_runtime_resume, NULL)
4031};
4032
4033static struct platform_driver tegra_xudc_driver = {
4034 .probe = tegra_xudc_probe,
4035 .remove = tegra_xudc_remove,
4036 .driver = {
4037 .name = "tegra-xudc",
4038 .pm = &tegra_xudc_pm_ops,
4039 .of_match_table = tegra_xudc_of_match,
4040 },
4041};
4042module_platform_driver(tegra_xudc_driver);
4043
4044MODULE_DESCRIPTION("NVIDIA Tegra XUSB Device Controller");
4045MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
4046MODULE_AUTHOR("Hui Fu <hfu@nvidia.com>");
4047MODULE_AUTHOR("Nagarjuna Kristam <nkristam@nvidia.com>");
4048MODULE_LICENSE("GPL v2");