1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 */
   4
   5#include <linux/gfp.h>
   6#include <linux/init.h>
   7#include <linux/ratelimit.h>
   8#include <linux/usb.h>
   9#include <linux/usb/audio.h>
  10#include <linux/slab.h>
  11
  12#include <sound/core.h>
  13#include <sound/pcm.h>
  14#include <sound/pcm_params.h>
  15
  16#include "usbaudio.h"
  17#include "helper.h"
  18#include "card.h"
  19#include "endpoint.h"
  20#include "pcm.h"
  21#include "clock.h"
  22#include "quirks.h"
  23
  24enum {
  25	EP_STATE_STOPPED,
  26	EP_STATE_RUNNING,
  27	EP_STATE_STOPPING,
  28};
  29
  30/* interface refcounting */
  31struct snd_usb_iface_ref {
  32	unsigned char iface;
  33	bool need_setup;
  34	int opened;
  35	int altset;
  36	struct list_head list;
  37};
  38
  39/* clock refcounting */
  40struct snd_usb_clock_ref {
  41	unsigned char clock;
  42	atomic_t locked;
  43	int opened;
  44	int rate;
  45	bool need_setup;
  46	struct list_head list;
  47};
  48
  49/*
  50 * snd_usb_endpoint is a model that abstracts everything related to an
  51 * USB endpoint and its streaming.
  52 *
  53 * There are functions to activate and deactivate the streaming URBs and
  54 * optional callbacks to let the pcm logic handle the actual content of the
  55 * packets for playback and record. Thus, the bus streaming and the audio
  56 * handlers are fully decoupled.
  57 *
  58 * There are two different types of endpoints in audio applications.
  59 *
  60 * SND_USB_ENDPOINT_TYPE_DATA handles full audio data payload for both
  61 * inbound and outbound traffic.
  62 *
  63 * SND_USB_ENDPOINT_TYPE_SYNC endpoints are for inbound traffic only and
  64 * expect the payload to carry Q10.14 / Q16.16 formatted sync information
  65 * (3 or 4 bytes).
  66 *
  67 * Each endpoint has to be configured prior to being used by calling
  68 * snd_usb_endpoint_set_params().
  69 *
  70 * The model incorporates a reference counting, so that multiple users
  71 * can call snd_usb_endpoint_start() and snd_usb_endpoint_stop(), and
  72 * only the first user will effectively start the URBs, and only the last
  73 * one to stop it will tear the URBs down again.
  74 */
  75
  76/*
  77 * convert a sampling rate into our full speed format (fs/1000 in Q16.16)
  78 * this will overflow at approx 524 kHz
  79 */
  80static inline unsigned get_usb_full_speed_rate(unsigned int rate)
  81{
  82	return ((rate << 13) + 62) / 125;
  83}
  84
  85/*
  86 * convert a sampling rate into USB high speed format (fs/8000 in Q16.16)
  87 * this will overflow at approx 4 MHz
  88 */
  89static inline unsigned get_usb_high_speed_rate(unsigned int rate)
  90{
  91	return ((rate << 10) + 62) / 125;
  92}
  93
  94/*
  95 * release a urb data
  96 */
  97static void release_urb_ctx(struct snd_urb_ctx *u)
  98{
  99	if (u->urb && u->buffer_size)
 100		usb_free_coherent(u->ep->chip->dev, u->buffer_size,
 101				  u->urb->transfer_buffer,
 102				  u->urb->transfer_dma);
 103	usb_free_urb(u->urb);
 104	u->urb = NULL;
 105	u->buffer_size = 0;
 106}
 107
 108static const char *usb_error_string(int err)
 109{
 110	switch (err) {
 111	case -ENODEV:
 112		return "no device";
 113	case -ENOENT:
 114		return "endpoint not enabled";
 115	case -EPIPE:
 116		return "endpoint stalled";
 117	case -ENOSPC:
 118		return "not enough bandwidth";
 119	case -ESHUTDOWN:
 120		return "device disabled";
 121	case -EHOSTUNREACH:
 122		return "device suspended";
 123	case -EINVAL:
 124	case -EAGAIN:
 125	case -EFBIG:
 126	case -EMSGSIZE:
 127		return "internal error";
 128	default:
 129		return "unknown error";
 130	}
 131}
 132
 133static inline bool ep_state_running(struct snd_usb_endpoint *ep)
 134{
 135	return atomic_read(&ep->state) == EP_STATE_RUNNING;
 136}
 137
 138static inline bool ep_state_update(struct snd_usb_endpoint *ep, int old, int new)
 139{
 140	return atomic_try_cmpxchg(&ep->state, &old, new);
 141}
 142
 143/**
 144 * snd_usb_endpoint_implicit_feedback_sink: Report endpoint usage type
 145 *
 146 * @ep: The snd_usb_endpoint
 147 *
 148 * Determine whether an endpoint is driven by an implicit feedback
 149 * data endpoint source.
 150 */
 151int snd_usb_endpoint_implicit_feedback_sink(struct snd_usb_endpoint *ep)
 152{
 153	return  ep->implicit_fb_sync && usb_pipeout(ep->pipe);
 154}
 155
 156/*
 157 * Return the number of samples to be sent in the next packet
 158 * for streaming based on information derived from sync endpoints
 159 *
 160 * This won't be used for implicit feedback which takes the packet size
 161 * returned from the sync source
 162 */
 163static int slave_next_packet_size(struct snd_usb_endpoint *ep,
 164				  unsigned int avail)
 165{
 166	unsigned long flags;
 167	unsigned int phase;
 168	int ret;
 169
 170	if (ep->fill_max)
 171		return ep->maxframesize;
 172
 173	spin_lock_irqsave(&ep->lock, flags);
 174	phase = (ep->phase & 0xffff) + (ep->freqm << ep->datainterval);
 175	ret = min(phase >> 16, ep->maxframesize);
 176	if (avail && ret >= avail)
 177		ret = -EAGAIN;
 178	else
 179		ep->phase = phase;
 180	spin_unlock_irqrestore(&ep->lock, flags);
 181
 182	return ret;
 183}
 184
 185/*
 186 * Return the number of samples to be sent in the next packet
 187 * for adaptive and synchronous endpoints
 188 */
 189static int next_packet_size(struct snd_usb_endpoint *ep, unsigned int avail)
 190{
 191	unsigned int sample_accum;
 192	int ret;
 193
 194	if (ep->fill_max)
 195		return ep->maxframesize;
 196
 197	sample_accum = ep->sample_accum + ep->sample_rem;
 198	if (sample_accum >= ep->pps) {
 199		sample_accum -= ep->pps;
 200		ret = ep->packsize[1];
 201	} else {
 202		ret = ep->packsize[0];
 203	}
 204	if (avail && ret >= avail)
 205		ret = -EAGAIN;
 206	else
 207		ep->sample_accum = sample_accum;
 208
 209	return ret;
 210}
 211
 212/*
 213 * snd_usb_endpoint_next_packet_size: Return the number of samples to be sent
 214 * in the next packet
 215 *
 216 * If the size is equal or exceeds @avail, don't proceed but return -EAGAIN
 217 * Exception: @avail = 0 for skipping the check.
 218 */
 219int snd_usb_endpoint_next_packet_size(struct snd_usb_endpoint *ep,
 220				      struct snd_urb_ctx *ctx, int idx,
 221				      unsigned int avail)
 222{
 223	unsigned int packet;
 224
 225	packet = ctx->packet_size[idx];
 226	if (packet) {
 227		if (avail && packet >= avail)
 228			return -EAGAIN;
 229		return packet;
 230	}
 231
 232	if (ep->sync_source)
 233		return slave_next_packet_size(ep, avail);
 234	else
 235		return next_packet_size(ep, avail);
 236}
 237
 238static void call_retire_callback(struct snd_usb_endpoint *ep,
 239				 struct urb *urb)
 240{
 241	struct snd_usb_substream *data_subs;
 242
 243	data_subs = READ_ONCE(ep->data_subs);
 244	if (data_subs && ep->retire_data_urb)
 245		ep->retire_data_urb(data_subs, urb);
 246}
 247
 248static void retire_outbound_urb(struct snd_usb_endpoint *ep,
 249				struct snd_urb_ctx *urb_ctx)
 250{
 251	call_retire_callback(ep, urb_ctx->urb);
 252}
 253
 254static void snd_usb_handle_sync_urb(struct snd_usb_endpoint *ep,
 255				    struct snd_usb_endpoint *sender,
 256				    const struct urb *urb);
 257
 258static void retire_inbound_urb(struct snd_usb_endpoint *ep,
 259			       struct snd_urb_ctx *urb_ctx)
 260{
 261	struct urb *urb = urb_ctx->urb;
 262	struct snd_usb_endpoint *sync_sink;
 263
 264	if (unlikely(ep->skip_packets > 0)) {
 265		ep->skip_packets--;
 266		return;
 267	}
 268
 269	sync_sink = READ_ONCE(ep->sync_sink);
 270	if (sync_sink)
 271		snd_usb_handle_sync_urb(sync_sink, ep, urb);
 272
 273	call_retire_callback(ep, urb);
 274}
 275
 276static inline bool has_tx_length_quirk(struct snd_usb_audio *chip)
 277{
 278	return chip->quirk_flags & QUIRK_FLAG_TX_LENGTH;
 279}
 280
 281static void prepare_silent_urb(struct snd_usb_endpoint *ep,
 282			       struct snd_urb_ctx *ctx)
 283{
 284	struct urb *urb = ctx->urb;
 285	unsigned int offs = 0;
 286	unsigned int extra = 0;
 287	__le32 packet_length;
 288	int i;
 289
 290	/* For tx_length_quirk, put packet length at start of packet */
 291	if (has_tx_length_quirk(ep->chip))
 292		extra = sizeof(packet_length);
 293
 294	for (i = 0; i < ctx->packets; ++i) {
 295		unsigned int offset;
 296		unsigned int length;
 297		int counts;
 298
 299		counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0);
 300		length = counts * ep->stride; /* number of silent bytes */
 301		offset = offs * ep->stride + extra * i;
 302		urb->iso_frame_desc[i].offset = offset;
 303		urb->iso_frame_desc[i].length = length + extra;
 304		if (extra) {
 305			packet_length = cpu_to_le32(length);
 306			memcpy(urb->transfer_buffer + offset,
 307			       &packet_length, sizeof(packet_length));
 308		}
 309		memset(urb->transfer_buffer + offset + extra,
 310		       ep->silence_value, length);
 311		offs += counts;
 312	}
 313
 314	urb->number_of_packets = ctx->packets;
 315	urb->transfer_buffer_length = offs * ep->stride + ctx->packets * extra;
 316	ctx->queued = 0;
 317}
 318
 319/*
 320 * Prepare a PLAYBACK urb for submission to the bus.
 321 */
 322static int prepare_outbound_urb(struct snd_usb_endpoint *ep,
 323				struct snd_urb_ctx *ctx,
 324				bool in_stream_lock)
 325{
 326	struct urb *urb = ctx->urb;
 327	unsigned char *cp = urb->transfer_buffer;
 328	struct snd_usb_substream *data_subs;
 329
 330	urb->dev = ep->chip->dev; /* we need to set this at each time */
 331
 332	switch (ep->type) {
 333	case SND_USB_ENDPOINT_TYPE_DATA:
 334		data_subs = READ_ONCE(ep->data_subs);
 335		if (data_subs && ep->prepare_data_urb)
 336			return ep->prepare_data_urb(data_subs, urb, in_stream_lock);
 337		/* no data provider, so send silence */
 338		prepare_silent_urb(ep, ctx);
 339		break;
 340
 341	case SND_USB_ENDPOINT_TYPE_SYNC:
 342		if (snd_usb_get_speed(ep->chip->dev) >= USB_SPEED_HIGH) {
 343			/*
 344			 * fill the length and offset of each urb descriptor.
 345			 * the fixed 12.13 frequency is passed as 16.16 through the pipe.
 346			 */
 347			urb->iso_frame_desc[0].length = 4;
 348			urb->iso_frame_desc[0].offset = 0;
 349			cp[0] = ep->freqn;
 350			cp[1] = ep->freqn >> 8;
 351			cp[2] = ep->freqn >> 16;
 352			cp[3] = ep->freqn >> 24;
 353		} else {
 354			/*
 355			 * fill the length and offset of each urb descriptor.
 356			 * the fixed 10.14 frequency is passed through the pipe.
 357			 */
 358			urb->iso_frame_desc[0].length = 3;
 359			urb->iso_frame_desc[0].offset = 0;
 360			cp[0] = ep->freqn >> 2;
 361			cp[1] = ep->freqn >> 10;
 362			cp[2] = ep->freqn >> 18;
 363		}
 364
 365		break;
 366	}
 367	return 0;
 368}
 369
 370/*
 371 * Prepare a CAPTURE or SYNC urb for submission to the bus.
 372 */
 373static int prepare_inbound_urb(struct snd_usb_endpoint *ep,
 374			       struct snd_urb_ctx *urb_ctx)
 375{
 376	int i, offs;
 377	struct urb *urb = urb_ctx->urb;
 378
 379	urb->dev = ep->chip->dev; /* we need to set this at each time */
 380
 381	switch (ep->type) {
 382	case SND_USB_ENDPOINT_TYPE_DATA:
 383		offs = 0;
 384		for (i = 0; i < urb_ctx->packets; i++) {
 385			urb->iso_frame_desc[i].offset = offs;
 386			urb->iso_frame_desc[i].length = ep->curpacksize;
 387			offs += ep->curpacksize;
 388		}
 389
 390		urb->transfer_buffer_length = offs;
 391		urb->number_of_packets = urb_ctx->packets;
 392		break;
 393
 394	case SND_USB_ENDPOINT_TYPE_SYNC:
 395		urb->iso_frame_desc[0].length = min(4u, ep->syncmaxsize);
 396		urb->iso_frame_desc[0].offset = 0;
 397		break;
 398	}
 399	return 0;
 400}
 401
 402/* notify an error as XRUN to the assigned PCM data substream */
 403static void notify_xrun(struct snd_usb_endpoint *ep)
 404{
 405	struct snd_usb_substream *data_subs;
 406	struct snd_pcm_substream *psubs;
 407
 408	data_subs = READ_ONCE(ep->data_subs);
 409	if (!data_subs)
 410		return;
 411	psubs = data_subs->pcm_substream;
 412	if (psubs && psubs->runtime &&
 413	    psubs->runtime->state == SNDRV_PCM_STATE_RUNNING)
 414		snd_pcm_stop_xrun(psubs);
 415}
 416
 417static struct snd_usb_packet_info *
 418next_packet_fifo_enqueue(struct snd_usb_endpoint *ep)
 419{
 420	struct snd_usb_packet_info *p;
 421
 422	p = ep->next_packet + (ep->next_packet_head + ep->next_packet_queued) %
 423		ARRAY_SIZE(ep->next_packet);
 424	ep->next_packet_queued++;
 425	return p;
 426}
 427
 428static struct snd_usb_packet_info *
 429next_packet_fifo_dequeue(struct snd_usb_endpoint *ep)
 430{
 431	struct snd_usb_packet_info *p;
 432
 433	p = ep->next_packet + ep->next_packet_head;
 434	ep->next_packet_head++;
 435	ep->next_packet_head %= ARRAY_SIZE(ep->next_packet);
 436	ep->next_packet_queued--;
 437	return p;
 438}
 439
 440static void push_back_to_ready_list(struct snd_usb_endpoint *ep,
 441				    struct snd_urb_ctx *ctx)
 442{
 443	unsigned long flags;
 444
 445	spin_lock_irqsave(&ep->lock, flags);
 446	list_add_tail(&ctx->ready_list, &ep->ready_playback_urbs);
 447	spin_unlock_irqrestore(&ep->lock, flags);
 448}
 449
 450/*
 451 * Send output urbs that have been prepared previously. URBs are dequeued
 452 * from ep->ready_playback_urbs and in case there aren't any available
 453 * or there are no packets that have been prepared, this function does
 454 * nothing.
 455 *
 456 * The reason why the functionality of sending and preparing URBs is separated
 457 * is that host controllers don't guarantee the order in which they return
 458 * inbound and outbound packets to their submitters.
 459 *
 460 * This function is used both for implicit feedback endpoints and in low-
 461 * latency playback mode.
 462 */
 463int snd_usb_queue_pending_output_urbs(struct snd_usb_endpoint *ep,
 464				      bool in_stream_lock)
 465{
 466	bool implicit_fb = snd_usb_endpoint_implicit_feedback_sink(ep);
 467
 468	while (ep_state_running(ep)) {
 469
 470		unsigned long flags;
 471		struct snd_usb_packet_info *packet;
 472		struct snd_urb_ctx *ctx = NULL;
 473		int err, i;
 474
 475		spin_lock_irqsave(&ep->lock, flags);
 476		if ((!implicit_fb || ep->next_packet_queued > 0) &&
 477		    !list_empty(&ep->ready_playback_urbs)) {
 478			/* take URB out of FIFO */
 479			ctx = list_first_entry(&ep->ready_playback_urbs,
 480					       struct snd_urb_ctx, ready_list);
 481			list_del_init(&ctx->ready_list);
 482			if (implicit_fb)
 483				packet = next_packet_fifo_dequeue(ep);
 484		}
 485		spin_unlock_irqrestore(&ep->lock, flags);
 486
 487		if (ctx == NULL)
 488			break;
 489
 490		/* copy over the length information */
 491		if (implicit_fb) {
 492			for (i = 0; i < packet->packets; i++)
 493				ctx->packet_size[i] = packet->packet_size[i];
 494		}
 495
 496		/* call the data handler to fill in playback data */
 497		err = prepare_outbound_urb(ep, ctx, in_stream_lock);
 498		/* can be stopped during prepare callback */
 499		if (unlikely(!ep_state_running(ep)))
 500			break;
 501		if (err < 0) {
 502			/* push back to ready list again for -EAGAIN */
 503			if (err == -EAGAIN) {
 504				push_back_to_ready_list(ep, ctx);
 505				break;
 506			}
 507
 508			if (!in_stream_lock)
 509				notify_xrun(ep);
 510			return -EPIPE;
 511		}
 512
 513		if (!atomic_read(&ep->chip->shutdown))
 514			err = usb_submit_urb(ctx->urb, GFP_ATOMIC);
 515		else
 516			err = -ENODEV;
 517		if (err < 0) {
 518			if (!atomic_read(&ep->chip->shutdown)) {
 519				usb_audio_err(ep->chip,
 520					      "Unable to submit urb #%d: %d at %s\n",
 521					      ctx->index, err, __func__);
 522				if (!in_stream_lock)
 523					notify_xrun(ep);
 524			}
 525			return -EPIPE;
 526		}
 527
 528		set_bit(ctx->index, &ep->active_mask);
 529		atomic_inc(&ep->submitted_urbs);
 530	}
 531
 532	return 0;
 533}
 534
 535/*
 536 * complete callback for urbs
 537 */
 538static void snd_complete_urb(struct urb *urb)
 539{
 540	struct snd_urb_ctx *ctx = urb->context;
 541	struct snd_usb_endpoint *ep = ctx->ep;
 542	int err;
 543
 544	if (unlikely(urb->status == -ENOENT ||		/* unlinked */
 545		     urb->status == -ENODEV ||		/* device removed */
 546		     urb->status == -ECONNRESET ||	/* unlinked */
 547		     urb->status == -ESHUTDOWN))	/* device disabled */
 548		goto exit_clear;
 549	/* device disconnected */
 550	if (unlikely(atomic_read(&ep->chip->shutdown)))
 551		goto exit_clear;
 552
 553	if (unlikely(!ep_state_running(ep)))
 554		goto exit_clear;
 555
 556	if (usb_pipeout(ep->pipe)) {
 557		retire_outbound_urb(ep, ctx);
 558		/* can be stopped during retire callback */
 559		if (unlikely(!ep_state_running(ep)))
 560			goto exit_clear;
 561
 562		/* in low-latency and implicit-feedback modes, push back the
 563		 * URB to ready list at first, then process as much as possible
 564		 */
 565		if (ep->lowlatency_playback ||
 566		     snd_usb_endpoint_implicit_feedback_sink(ep)) {
 567			push_back_to_ready_list(ep, ctx);
 568			clear_bit(ctx->index, &ep->active_mask);
 569			snd_usb_queue_pending_output_urbs(ep, false);
 570			/* decrement at last, and check xrun */
 571			if (atomic_dec_and_test(&ep->submitted_urbs) &&
 572			    !snd_usb_endpoint_implicit_feedback_sink(ep))
 573				notify_xrun(ep);
 574			return;
 575		}
 576
 577		/* in non-lowlatency mode, no error handling for prepare */
 578		prepare_outbound_urb(ep, ctx, false);
 579		/* can be stopped during prepare callback */
 580		if (unlikely(!ep_state_running(ep)))
 581			goto exit_clear;
 582	} else {
 583		retire_inbound_urb(ep, ctx);
 584		/* can be stopped during retire callback */
 585		if (unlikely(!ep_state_running(ep)))
 586			goto exit_clear;
 587
 588		prepare_inbound_urb(ep, ctx);
 589	}
 590
 591	if (!atomic_read(&ep->chip->shutdown))
 592		err = usb_submit_urb(urb, GFP_ATOMIC);
 593	else
 594		err = -ENODEV;
 595	if (err == 0)
 596		return;
 597
 598	if (!atomic_read(&ep->chip->shutdown)) {
 599		usb_audio_err(ep->chip, "cannot submit urb (err = %d)\n", err);
 600		notify_xrun(ep);
 601	}
 602
 603exit_clear:
 604	clear_bit(ctx->index, &ep->active_mask);
 605	atomic_dec(&ep->submitted_urbs);
 606}
 607
 608/*
 609 * Find or create a refcount object for the given interface
 610 *
 611 * The objects are released altogether in snd_usb_endpoint_free_all()
 612 */
 613static struct snd_usb_iface_ref *
 614iface_ref_find(struct snd_usb_audio *chip, int iface)
 615{
 616	struct snd_usb_iface_ref *ip;
 617
 618	list_for_each_entry(ip, &chip->iface_ref_list, list)
 619		if (ip->iface == iface)
 620			return ip;
 621
 622	ip = kzalloc(sizeof(*ip), GFP_KERNEL);
 623	if (!ip)
 624		return NULL;
 625	ip->iface = iface;
 626	list_add_tail(&ip->list, &chip->iface_ref_list);
 627	return ip;
 628}
 629
 630/* Similarly, a refcount object for clock */
 631static struct snd_usb_clock_ref *
 632clock_ref_find(struct snd_usb_audio *chip, int clock)
 633{
 634	struct snd_usb_clock_ref *ref;
 635
 636	list_for_each_entry(ref, &chip->clock_ref_list, list)
 637		if (ref->clock == clock)
 638			return ref;
 639
 640	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
 641	if (!ref)
 642		return NULL;
 643	ref->clock = clock;
 644	atomic_set(&ref->locked, 0);
 645	list_add_tail(&ref->list, &chip->clock_ref_list);
 646	return ref;
 647}
 648
 649/*
 650 * Get the existing endpoint object corresponding EP
 651 * Returns NULL if not present.
 652 */
 653struct snd_usb_endpoint *
 654snd_usb_get_endpoint(struct snd_usb_audio *chip, int ep_num)
 655{
 656	struct snd_usb_endpoint *ep;
 657
 658	list_for_each_entry(ep, &chip->ep_list, list) {
 659		if (ep->ep_num == ep_num)
 660			return ep;
 661	}
 662
 663	return NULL;
 664}
 665
 666#define ep_type_name(type) \
 667	(type == SND_USB_ENDPOINT_TYPE_DATA ? "data" : "sync")
 668
 669/**
 670 * snd_usb_add_endpoint: Add an endpoint to an USB audio chip
 671 *
 672 * @chip: The chip
 673 * @ep_num: The number of the endpoint to use
 674 * @type: SND_USB_ENDPOINT_TYPE_DATA or SND_USB_ENDPOINT_TYPE_SYNC
 675 *
 676 * If the requested endpoint has not been added to the given chip before,
 677 * a new instance is created.
 678 *
 679 * Returns zero on success or a negative error code.
 680 *
 681 * New endpoints will be added to chip->ep_list and freed by
 682 * calling snd_usb_endpoint_free_all().
 683 *
 684 * For SND_USB_ENDPOINT_TYPE_SYNC, the caller needs to guarantee that
 685 * bNumEndpoints > 1 beforehand.
 686 */
 687int snd_usb_add_endpoint(struct snd_usb_audio *chip, int ep_num, int type)
 688{
 689	struct snd_usb_endpoint *ep;
 690	bool is_playback;
 691
 692	ep = snd_usb_get_endpoint(chip, ep_num);
 693	if (ep)
 694		return 0;
 695
 696	usb_audio_dbg(chip, "Creating new %s endpoint #%x\n",
 697		      ep_type_name(type),
 698		      ep_num);
 699	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
 700	if (!ep)
 701		return -ENOMEM;
 702
 703	ep->chip = chip;
 704	spin_lock_init(&ep->lock);
 705	ep->type = type;
 706	ep->ep_num = ep_num;
 707	INIT_LIST_HEAD(&ep->ready_playback_urbs);
 708	atomic_set(&ep->submitted_urbs, 0);
 709
 710	is_playback = ((ep_num & USB_ENDPOINT_DIR_MASK) == USB_DIR_OUT);
 711	ep_num &= USB_ENDPOINT_NUMBER_MASK;
 712	if (is_playback)
 713		ep->pipe = usb_sndisocpipe(chip->dev, ep_num);
 714	else
 715		ep->pipe = usb_rcvisocpipe(chip->dev, ep_num);
 716
 717	list_add_tail(&ep->list, &chip->ep_list);
 718	return 0;
 719}
 720
 721/* Set up syncinterval and maxsyncsize for a sync EP */
 722static void endpoint_set_syncinterval(struct snd_usb_audio *chip,
 723				      struct snd_usb_endpoint *ep)
 724{
 725	struct usb_host_interface *alts;
 726	struct usb_endpoint_descriptor *desc;
 727
 728	alts = snd_usb_get_host_interface(chip, ep->iface, ep->altsetting);
 729	if (!alts)
 730		return;
 731
 732	desc = get_endpoint(alts, ep->ep_idx);
 733	if (desc->bLength >= USB_DT_ENDPOINT_AUDIO_SIZE &&
 734	    desc->bRefresh >= 1 && desc->bRefresh <= 9)
 735		ep->syncinterval = desc->bRefresh;
 736	else if (snd_usb_get_speed(chip->dev) == USB_SPEED_FULL)
 737		ep->syncinterval = 1;
 738	else if (desc->bInterval >= 1 && desc->bInterval <= 16)
 739		ep->syncinterval = desc->bInterval - 1;
 740	else
 741		ep->syncinterval = 3;
 742
 743	ep->syncmaxsize = le16_to_cpu(desc->wMaxPacketSize);
 744}
 745
 746static bool endpoint_compatible(struct snd_usb_endpoint *ep,
 747				const struct audioformat *fp,
 748				const struct snd_pcm_hw_params *params)
 749{
 750	if (!ep->opened)
 751		return false;
 752	if (ep->cur_audiofmt != fp)
 753		return false;
 754	if (ep->cur_rate != params_rate(params) ||
 755	    ep->cur_format != params_format(params) ||
 756	    ep->cur_period_frames != params_period_size(params) ||
 757	    ep->cur_buffer_periods != params_periods(params))
 758		return false;
 759	return true;
 760}
 761
 762/*
 763 * Check whether the given fp and hw params are compatible with the current
 764 * setup of the target EP for implicit feedback sync
 765 */
 766bool snd_usb_endpoint_compatible(struct snd_usb_audio *chip,
 767				 struct snd_usb_endpoint *ep,
 768				 const struct audioformat *fp,
 769				 const struct snd_pcm_hw_params *params)
 770{
 771	bool ret;
 772
 773	mutex_lock(&chip->mutex);
 774	ret = endpoint_compatible(ep, fp, params);
 775	mutex_unlock(&chip->mutex);
 776	return ret;
 777}
 778
 779/*
 780 * snd_usb_endpoint_open: Open the endpoint
 781 *
 782 * Called from hw_params to assign the endpoint to the substream.
 783 * It's reference-counted, and only the first opener is allowed to set up
 784 * arbitrary parameters.  The later opener must be compatible with the
 785 * former opened parameters.
 786 * The endpoint needs to be closed via snd_usb_endpoint_close() later.
 787 *
 788 * Note that this function doesn't configure the endpoint.  The substream
 789 * needs to set it up later via snd_usb_endpoint_set_params() and
 790 * snd_usb_endpoint_prepare().
 791 */
 792struct snd_usb_endpoint *
 793snd_usb_endpoint_open(struct snd_usb_audio *chip,
 794		      const struct audioformat *fp,
 795		      const struct snd_pcm_hw_params *params,
 796		      bool is_sync_ep,
 797		      bool fixed_rate)
 798{
 799	struct snd_usb_endpoint *ep;
 800	int ep_num = is_sync_ep ? fp->sync_ep : fp->endpoint;
 801
 802	mutex_lock(&chip->mutex);
 803	ep = snd_usb_get_endpoint(chip, ep_num);
 804	if (!ep) {
 805		usb_audio_err(chip, "Cannot find EP 0x%x to open\n", ep_num);
 806		goto unlock;
 807	}
 808
 809	if (!ep->opened) {
 810		if (is_sync_ep) {
 811			ep->iface = fp->sync_iface;
 812			ep->altsetting = fp->sync_altsetting;
 813			ep->ep_idx = fp->sync_ep_idx;
 814		} else {
 815			ep->iface = fp->iface;
 816			ep->altsetting = fp->altsetting;
 817			ep->ep_idx = fp->ep_idx;
 818		}
 819		usb_audio_dbg(chip, "Open EP 0x%x, iface=%d:%d, idx=%d\n",
 820			      ep_num, ep->iface, ep->altsetting, ep->ep_idx);
 821
 822		ep->iface_ref = iface_ref_find(chip, ep->iface);
 823		if (!ep->iface_ref) {
 824			ep = NULL;
 825			goto unlock;
 826		}
 827
 828		if (fp->protocol != UAC_VERSION_1) {
 829			ep->clock_ref = clock_ref_find(chip, fp->clock);
 830			if (!ep->clock_ref) {
 831				ep = NULL;
 832				goto unlock;
 833			}
 834			ep->clock_ref->opened++;
 835		}
 836
 837		ep->cur_audiofmt = fp;
 838		ep->cur_channels = fp->channels;
 839		ep->cur_rate = params_rate(params);
 840		ep->cur_format = params_format(params);
 841		ep->cur_frame_bytes = snd_pcm_format_physical_width(ep->cur_format) *
 842			ep->cur_channels / 8;
 843		ep->cur_period_frames = params_period_size(params);
 844		ep->cur_period_bytes = ep->cur_period_frames * ep->cur_frame_bytes;
 845		ep->cur_buffer_periods = params_periods(params);
 846
 847		if (ep->type == SND_USB_ENDPOINT_TYPE_SYNC)
 848			endpoint_set_syncinterval(chip, ep);
 849
 850		ep->implicit_fb_sync = fp->implicit_fb;
 851		ep->need_setup = true;
 852		ep->need_prepare = true;
 853		ep->fixed_rate = fixed_rate;
 854
 855		usb_audio_dbg(chip, "  channels=%d, rate=%d, format=%s, period_bytes=%d, periods=%d, implicit_fb=%d\n",
 856			      ep->cur_channels, ep->cur_rate,
 857			      snd_pcm_format_name(ep->cur_format),
 858			      ep->cur_period_bytes, ep->cur_buffer_periods,
 859			      ep->implicit_fb_sync);
 860
 861	} else {
 862		if (WARN_ON(!ep->iface_ref)) {
 863			ep = NULL;
 864			goto unlock;
 865		}
 866
 867		if (!endpoint_compatible(ep, fp, params)) {
 868			usb_audio_err(chip, "Incompatible EP setup for 0x%x\n",
 869				      ep_num);
 870			ep = NULL;
 871			goto unlock;
 872		}
 873
 874		usb_audio_dbg(chip, "Reopened EP 0x%x (count %d)\n",
 875			      ep_num, ep->opened);
 876	}
 877
 878	if (!ep->iface_ref->opened++)
 879		ep->iface_ref->need_setup = true;
 880
 881	ep->opened++;
 882
 883 unlock:
 884	mutex_unlock(&chip->mutex);
 885	return ep;
 886}
 887
 888/*
 889 * snd_usb_endpoint_set_sync: Link data and sync endpoints
 890 *
 891 * Pass NULL to sync_ep to unlink again
 892 */
 893void snd_usb_endpoint_set_sync(struct snd_usb_audio *chip,
 894			       struct snd_usb_endpoint *data_ep,
 895			       struct snd_usb_endpoint *sync_ep)
 896{
 897	data_ep->sync_source = sync_ep;
 898}
 899
 900/*
 901 * Set data endpoint callbacks and the assigned data stream
 902 *
 903 * Called at PCM trigger and cleanups.
 904 * Pass NULL to deactivate each callback.
 905 */
 906void snd_usb_endpoint_set_callback(struct snd_usb_endpoint *ep,
 907				   int (*prepare)(struct snd_usb_substream *subs,
 908						  struct urb *urb,
 909						  bool in_stream_lock),
 910				   void (*retire)(struct snd_usb_substream *subs,
 911						  struct urb *urb),
 912				   struct snd_usb_substream *data_subs)
 913{
 914	ep->prepare_data_urb = prepare;
 915	ep->retire_data_urb = retire;
 916	if (data_subs)
 917		ep->lowlatency_playback = data_subs->lowlatency_playback;
 918	else
 919		ep->lowlatency_playback = false;
 920	WRITE_ONCE(ep->data_subs, data_subs);
 921}
 922
 923static int endpoint_set_interface(struct snd_usb_audio *chip,
 924				  struct snd_usb_endpoint *ep,
 925				  bool set)
 926{
 927	int altset = set ? ep->altsetting : 0;
 928	int err;
 929
 930	if (ep->iface_ref->altset == altset)
 931		return 0;
 932	/* already disconnected? */
 933	if (unlikely(atomic_read(&chip->shutdown)))
 934		return -ENODEV;
 935
 936	usb_audio_dbg(chip, "Setting usb interface %d:%d for EP 0x%x\n",
 937		      ep->iface, altset, ep->ep_num);
 938	err = usb_set_interface(chip->dev, ep->iface, altset);
 939	if (err < 0) {
 940		usb_audio_err_ratelimited(
 941			chip, "%d:%d: usb_set_interface failed (%d)\n",
 942			ep->iface, altset, err);
 943		return err;
 944	}
 945
 946	if (chip->quirk_flags & QUIRK_FLAG_IFACE_DELAY)
 947		msleep(50);
 948	ep->iface_ref->altset = altset;
 949	return 0;
 950}
 951
 952/*
 953 * snd_usb_endpoint_close: Close the endpoint
 954 *
 955 * Unreference the already opened endpoint via snd_usb_endpoint_open().
 956 */
 957void snd_usb_endpoint_close(struct snd_usb_audio *chip,
 958			    struct snd_usb_endpoint *ep)
 959{
 960	mutex_lock(&chip->mutex);
 961	usb_audio_dbg(chip, "Closing EP 0x%x (count %d)\n",
 962		      ep->ep_num, ep->opened);
 963
 964	if (!--ep->iface_ref->opened &&
 965		!(chip->quirk_flags & QUIRK_FLAG_IFACE_SKIP_CLOSE))
 966		endpoint_set_interface(chip, ep, false);
 967
 968	if (!--ep->opened) {
 969		if (ep->clock_ref) {
 970			if (!--ep->clock_ref->opened)
 971				ep->clock_ref->rate = 0;
 972		}
 973		ep->iface = 0;
 974		ep->altsetting = 0;
 975		ep->cur_audiofmt = NULL;
 976		ep->cur_rate = 0;
 977		ep->iface_ref = NULL;
 978		ep->clock_ref = NULL;
 979		usb_audio_dbg(chip, "EP 0x%x closed\n", ep->ep_num);
 980	}
 981	mutex_unlock(&chip->mutex);
 982}
 983
 984/* Prepare for suspening EP, called from the main suspend handler */
 985void snd_usb_endpoint_suspend(struct snd_usb_endpoint *ep)
 986{
 987	ep->need_prepare = true;
 988	if (ep->iface_ref)
 989		ep->iface_ref->need_setup = true;
 990	if (ep->clock_ref)
 991		ep->clock_ref->rate = 0;
 992}
 993
 994/*
 995 *  wait until all urbs are processed.
 996 */
 997static int wait_clear_urbs(struct snd_usb_endpoint *ep)
 998{
 999	unsigned long end_time = jiffies + msecs_to_jiffies(1000);
1000	int alive;
1001
1002	if (atomic_read(&ep->state) != EP_STATE_STOPPING)
1003		return 0;
1004
1005	do {
1006		alive = atomic_read(&ep->submitted_urbs);
1007		if (!alive)
1008			break;
1009
1010		schedule_timeout_uninterruptible(1);
1011	} while (time_before(jiffies, end_time));
1012
1013	if (alive)
1014		usb_audio_err(ep->chip,
1015			"timeout: still %d active urbs on EP #%x\n",
1016			alive, ep->ep_num);
1017
1018	if (ep_state_update(ep, EP_STATE_STOPPING, EP_STATE_STOPPED)) {
1019		ep->sync_sink = NULL;
1020		snd_usb_endpoint_set_callback(ep, NULL, NULL, NULL);
1021	}
1022
1023	return 0;
1024}
1025
1026/* sync the pending stop operation;
1027 * this function itself doesn't trigger the stop operation
1028 */
1029void snd_usb_endpoint_sync_pending_stop(struct snd_usb_endpoint *ep)
1030{
1031	if (ep)
1032		wait_clear_urbs(ep);
1033}
1034
1035/*
1036 * Stop active urbs
1037 *
1038 * This function moves the EP to STOPPING state if it's being RUNNING.
1039 */
1040static int stop_urbs(struct snd_usb_endpoint *ep, bool force, bool keep_pending)
1041{
1042	unsigned int i;
1043	unsigned long flags;
1044
1045	if (!force && atomic_read(&ep->running))
1046		return -EBUSY;
1047
1048	if (!ep_state_update(ep, EP_STATE_RUNNING, EP_STATE_STOPPING))
1049		return 0;
1050
1051	spin_lock_irqsave(&ep->lock, flags);
1052	INIT_LIST_HEAD(&ep->ready_playback_urbs);
1053	ep->next_packet_head = 0;
1054	ep->next_packet_queued = 0;
1055	spin_unlock_irqrestore(&ep->lock, flags);
1056
1057	if (keep_pending)
1058		return 0;
1059
1060	for (i = 0; i < ep->nurbs; i++) {
1061		if (test_bit(i, &ep->active_mask)) {
1062			if (!test_and_set_bit(i, &ep->unlink_mask)) {
1063				struct urb *u = ep->urb[i].urb;
1064				usb_unlink_urb(u);
1065			}
1066		}
1067	}
1068
1069	return 0;
1070}
1071
1072/*
1073 * release an endpoint's urbs
1074 */
1075static int release_urbs(struct snd_usb_endpoint *ep, bool force)
1076{
1077	int i, err;
1078
1079	/* route incoming urbs to nirvana */
1080	snd_usb_endpoint_set_callback(ep, NULL, NULL, NULL);
1081
1082	/* stop and unlink urbs */
1083	err = stop_urbs(ep, force, false);
1084	if (err)
1085		return err;
1086
1087	wait_clear_urbs(ep);
1088
1089	for (i = 0; i < ep->nurbs; i++)
1090		release_urb_ctx(&ep->urb[i]);
1091
1092	usb_free_coherent(ep->chip->dev, SYNC_URBS * 4,
1093			  ep->syncbuf, ep->sync_dma);
1094
1095	ep->syncbuf = NULL;
1096	ep->nurbs = 0;
1097	return 0;
1098}
1099
1100/*
1101 * configure a data endpoint
1102 */
1103static int data_ep_set_params(struct snd_usb_endpoint *ep)
1104{
1105	struct snd_usb_audio *chip = ep->chip;
1106	unsigned int maxsize, minsize, packs_per_ms, max_packs_per_urb;
1107	unsigned int max_packs_per_period, urbs_per_period, urb_packs;
1108	unsigned int max_urbs, i;
1109	const struct audioformat *fmt = ep->cur_audiofmt;
1110	int frame_bits = ep->cur_frame_bytes * 8;
1111	int tx_length_quirk = (has_tx_length_quirk(chip) &&
1112			       usb_pipeout(ep->pipe));
1113
1114	usb_audio_dbg(chip, "Setting params for data EP 0x%x, pipe 0x%x\n",
1115		      ep->ep_num, ep->pipe);
1116
1117	if (ep->cur_format == SNDRV_PCM_FORMAT_DSD_U16_LE && fmt->dsd_dop) {
1118		/*
1119		 * When operating in DSD DOP mode, the size of a sample frame
1120		 * in hardware differs from the actual physical format width
1121		 * because we need to make room for the DOP markers.
1122		 */
1123		frame_bits += ep->cur_channels << 3;
1124	}
1125
1126	ep->datainterval = fmt->datainterval;
1127	ep->stride = frame_bits >> 3;
1128
1129	switch (ep->cur_format) {
1130	case SNDRV_PCM_FORMAT_U8:
1131		ep->silence_value = 0x80;
1132		break;
1133	case SNDRV_PCM_FORMAT_DSD_U8:
1134	case SNDRV_PCM_FORMAT_DSD_U16_LE:
1135	case SNDRV_PCM_FORMAT_DSD_U32_LE:
1136	case SNDRV_PCM_FORMAT_DSD_U16_BE:
1137	case SNDRV_PCM_FORMAT_DSD_U32_BE:
1138		ep->silence_value = 0x69;
1139		break;
1140	default:
1141		ep->silence_value = 0;
1142	}
1143
1144	/* assume max. frequency is 50% higher than nominal */
1145	ep->freqmax = ep->freqn + (ep->freqn >> 1);
1146	/* Round up freqmax to nearest integer in order to calculate maximum
1147	 * packet size, which must represent a whole number of frames.
1148	 * This is accomplished by adding 0x0.ffff before converting the
1149	 * Q16.16 format into integer.
1150	 * In order to accurately calculate the maximum packet size when
1151	 * the data interval is more than 1 (i.e. ep->datainterval > 0),
1152	 * multiply by the data interval prior to rounding. For instance,
1153	 * a freqmax of 41 kHz will result in a max packet size of 6 (5.125)
1154	 * frames with a data interval of 1, but 11 (10.25) frames with a
1155	 * data interval of 2.
1156	 * (ep->freqmax << ep->datainterval overflows at 8.192 MHz for the
1157	 * maximum datainterval value of 3, at USB full speed, higher for
1158	 * USB high speed, noting that ep->freqmax is in units of
1159	 * frames per packet in Q16.16 format.)
1160	 */
1161	maxsize = (((ep->freqmax << ep->datainterval) + 0xffff) >> 16) *
1162			 (frame_bits >> 3);
1163	if (tx_length_quirk)
1164		maxsize += sizeof(__le32); /* Space for length descriptor */
1165	/* but wMaxPacketSize might reduce this */
1166	if (ep->maxpacksize && ep->maxpacksize < maxsize) {
1167		/* whatever fits into a max. size packet */
1168		unsigned int data_maxsize = maxsize = ep->maxpacksize;
1169
1170		if (tx_length_quirk)
1171			/* Need to remove the length descriptor to calc freq */
1172			data_maxsize -= sizeof(__le32);
1173		ep->freqmax = (data_maxsize / (frame_bits >> 3))
1174				<< (16 - ep->datainterval);
1175	}
1176
1177	if (ep->fill_max)
1178		ep->curpacksize = ep->maxpacksize;
1179	else
1180		ep->curpacksize = maxsize;
1181
1182	if (snd_usb_get_speed(chip->dev) != USB_SPEED_FULL) {
1183		packs_per_ms = 8 >> ep->datainterval;
1184		max_packs_per_urb = MAX_PACKS_HS;
1185	} else {
1186		packs_per_ms = 1;
1187		max_packs_per_urb = MAX_PACKS;
1188	}
1189	if (ep->sync_source && !ep->implicit_fb_sync)
1190		max_packs_per_urb = min(max_packs_per_urb,
1191					1U << ep->sync_source->syncinterval);
1192	max_packs_per_urb = max(1u, max_packs_per_urb >> ep->datainterval);
1193
1194	/*
1195	 * Capture endpoints need to use small URBs because there's no way
1196	 * to tell in advance where the next period will end, and we don't
1197	 * want the next URB to complete much after the period ends.
1198	 *
1199	 * Playback endpoints with implicit sync much use the same parameters
1200	 * as their corresponding capture endpoint.
1201	 */
1202	if (usb_pipein(ep->pipe) || ep->implicit_fb_sync) {
1203
1204		/* make capture URBs <= 1 ms and smaller than a period */
1205		urb_packs = min(max_packs_per_urb, packs_per_ms);
1206		while (urb_packs > 1 && urb_packs * maxsize >= ep->cur_period_bytes)
1207			urb_packs >>= 1;
1208		ep->nurbs = MAX_URBS;
1209
1210	/*
1211	 * Playback endpoints without implicit sync are adjusted so that
1212	 * a period fits as evenly as possible in the smallest number of
1213	 * URBs.  The total number of URBs is adjusted to the size of the
1214	 * ALSA buffer, subject to the MAX_URBS and MAX_QUEUE limits.
1215	 */
1216	} else {
1217		/* determine how small a packet can be */
1218		minsize = (ep->freqn >> (16 - ep->datainterval)) *
1219				(frame_bits >> 3);
1220		/* with sync from device, assume it can be 12% lower */
1221		if (ep->sync_source)
1222			minsize -= minsize >> 3;
1223		minsize = max(minsize, 1u);
1224
1225		/* how many packets will contain an entire ALSA period? */
1226		max_packs_per_period = DIV_ROUND_UP(ep->cur_period_bytes, minsize);
1227
1228		/* how many URBs will contain a period? */
1229		urbs_per_period = DIV_ROUND_UP(max_packs_per_period,
1230				max_packs_per_urb);
1231		/* how many packets are needed in each URB? */
1232		urb_packs = DIV_ROUND_UP(max_packs_per_period, urbs_per_period);
1233
1234		/* limit the number of frames in a single URB */
1235		ep->max_urb_frames = DIV_ROUND_UP(ep->cur_period_frames,
1236						  urbs_per_period);
1237
1238		/* try to use enough URBs to contain an entire ALSA buffer */
1239		max_urbs = min((unsigned) MAX_URBS,
1240				MAX_QUEUE * packs_per_ms / urb_packs);
1241		ep->nurbs = min(max_urbs, urbs_per_period * ep->cur_buffer_periods);
1242	}
1243
1244	/* allocate and initialize data urbs */
1245	for (i = 0; i < ep->nurbs; i++) {
1246		struct snd_urb_ctx *u = &ep->urb[i];
1247		u->index = i;
1248		u->ep = ep;
1249		u->packets = urb_packs;
1250		u->buffer_size = maxsize * u->packets;
1251
1252		if (fmt->fmt_type == UAC_FORMAT_TYPE_II)
1253			u->packets++; /* for transfer delimiter */
1254		u->urb = usb_alloc_urb(u->packets, GFP_KERNEL);
1255		if (!u->urb)
1256			goto out_of_memory;
1257
1258		u->urb->transfer_buffer =
1259			usb_alloc_coherent(chip->dev, u->buffer_size,
1260					   GFP_KERNEL, &u->urb->transfer_dma);
1261		if (!u->urb->transfer_buffer)
1262			goto out_of_memory;
1263		u->urb->pipe = ep->pipe;
1264		u->urb->transfer_flags = URB_NO_TRANSFER_DMA_MAP;
1265		u->urb->interval = 1 << ep->datainterval;
1266		u->urb->context = u;
1267		u->urb->complete = snd_complete_urb;
1268		INIT_LIST_HEAD(&u->ready_list);
1269	}
1270
1271	return 0;
1272
1273out_of_memory:
1274	release_urbs(ep, false);
1275	return -ENOMEM;
1276}
1277
1278/*
1279 * configure a sync endpoint
1280 */
1281static int sync_ep_set_params(struct snd_usb_endpoint *ep)
1282{
1283	struct snd_usb_audio *chip = ep->chip;
1284	int i;
1285
1286	usb_audio_dbg(chip, "Setting params for sync EP 0x%x, pipe 0x%x\n",
1287		      ep->ep_num, ep->pipe);
1288
1289	ep->syncbuf = usb_alloc_coherent(chip->dev, SYNC_URBS * 4,
1290					 GFP_KERNEL, &ep->sync_dma);
1291	if (!ep->syncbuf)
1292		return -ENOMEM;
1293
1294	ep->nurbs = SYNC_URBS;
1295	for (i = 0; i < SYNC_URBS; i++) {
1296		struct snd_urb_ctx *u = &ep->urb[i];
1297		u->index = i;
1298		u->ep = ep;
1299		u->packets = 1;
1300		u->urb = usb_alloc_urb(1, GFP_KERNEL);
1301		if (!u->urb)
1302			goto out_of_memory;
1303		u->urb->transfer_buffer = ep->syncbuf + i * 4;
1304		u->urb->transfer_dma = ep->sync_dma + i * 4;
1305		u->urb->transfer_buffer_length = 4;
1306		u->urb->pipe = ep->pipe;
1307		u->urb->transfer_flags = URB_NO_TRANSFER_DMA_MAP;
1308		u->urb->number_of_packets = 1;
1309		u->urb->interval = 1 << ep->syncinterval;
1310		u->urb->context = u;
1311		u->urb->complete = snd_complete_urb;
1312	}
1313
1314	return 0;
1315
1316out_of_memory:
1317	release_urbs(ep, false);
1318	return -ENOMEM;
1319}
1320
1321/* update the rate of the referred clock; return the actual rate */
1322static int update_clock_ref_rate(struct snd_usb_audio *chip,
1323				 struct snd_usb_endpoint *ep)
1324{
1325	struct snd_usb_clock_ref *clock = ep->clock_ref;
1326	int rate = ep->cur_rate;
1327
1328	if (!clock || clock->rate == rate)
1329		return rate;
1330	if (clock->rate) {
1331		if (atomic_read(&clock->locked))
1332			return clock->rate;
1333		if (clock->rate != rate) {
1334			usb_audio_err(chip, "Mismatched sample rate %d vs %d for EP 0x%x\n",
1335				      clock->rate, rate, ep->ep_num);
1336			return clock->rate;
1337		}
1338	}
1339	clock->rate = rate;
1340	clock->need_setup = true;
1341	return rate;
1342}
1343
1344/*
1345 * snd_usb_endpoint_set_params: configure an snd_usb_endpoint
1346 *
1347 * It's called either from hw_params callback.
1348 * Determine the number of URBs to be used on this endpoint.
1349 * An endpoint must be configured before it can be started.
1350 * An endpoint that is already running can not be reconfigured.
1351 */
1352int snd_usb_endpoint_set_params(struct snd_usb_audio *chip,
1353				struct snd_usb_endpoint *ep)
1354{
1355	const struct audioformat *fmt = ep->cur_audiofmt;
1356	int err = 0;
1357
1358	mutex_lock(&chip->mutex);
1359	if (!ep->need_setup)
1360		goto unlock;
1361
1362	/* release old buffers, if any */
1363	err = release_urbs(ep, false);
1364	if (err < 0)
1365		goto unlock;
1366
1367	ep->datainterval = fmt->datainterval;
1368	ep->maxpacksize = fmt->maxpacksize;
1369	ep->fill_max = !!(fmt->attributes & UAC_EP_CS_ATTR_FILL_MAX);
1370
1371	if (snd_usb_get_speed(chip->dev) == USB_SPEED_FULL) {
1372		ep->freqn = get_usb_full_speed_rate(ep->cur_rate);
1373		ep->pps = 1000 >> ep->datainterval;
1374	} else {
1375		ep->freqn = get_usb_high_speed_rate(ep->cur_rate);
1376		ep->pps = 8000 >> ep->datainterval;
1377	}
1378
1379	ep->sample_rem = ep->cur_rate % ep->pps;
1380	ep->packsize[0] = ep->cur_rate / ep->pps;
1381	ep->packsize[1] = (ep->cur_rate + (ep->pps - 1)) / ep->pps;
1382
1383	/* calculate the frequency in 16.16 format */
1384	ep->freqm = ep->freqn;
1385	ep->freqshift = INT_MIN;
1386
1387	ep->phase = 0;
1388
1389	switch (ep->type) {
1390	case  SND_USB_ENDPOINT_TYPE_DATA:
1391		err = data_ep_set_params(ep);
1392		break;
1393	case  SND_USB_ENDPOINT_TYPE_SYNC:
1394		err = sync_ep_set_params(ep);
1395		break;
1396	default:
1397		err = -EINVAL;
1398	}
1399
1400	usb_audio_dbg(chip, "Set up %d URBS, ret=%d\n", ep->nurbs, err);
1401
1402	if (err < 0)
1403		goto unlock;
1404
1405	/* some unit conversions in runtime */
1406	ep->maxframesize = ep->maxpacksize / ep->cur_frame_bytes;
1407	ep->curframesize = ep->curpacksize / ep->cur_frame_bytes;
1408
1409	err = update_clock_ref_rate(chip, ep);
1410	if (err >= 0) {
1411		ep->need_setup = false;
1412		err = 0;
1413	}
1414
1415 unlock:
1416	mutex_unlock(&chip->mutex);
1417	return err;
1418}
1419
1420static int init_sample_rate(struct snd_usb_audio *chip,
1421			    struct snd_usb_endpoint *ep)
1422{
1423	struct snd_usb_clock_ref *clock = ep->clock_ref;
1424	int rate, err;
1425
1426	rate = update_clock_ref_rate(chip, ep);
1427	if (rate < 0)
1428		return rate;
1429	if (clock && !clock->need_setup)
1430		return 0;
1431
1432	if (!ep->fixed_rate) {
1433		err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, rate);
1434		if (err < 0) {
1435			if (clock)
1436				clock->rate = 0; /* reset rate */
1437			return err;
1438		}
1439	}
1440
1441	if (clock)
1442		clock->need_setup = false;
1443	return 0;
1444}
1445
1446/*
1447 * snd_usb_endpoint_prepare: Prepare the endpoint
1448 *
1449 * This function sets up the EP to be fully usable state.
1450 * It's called either from prepare callback.
1451 * The function checks need_setup flag, and performs nothing unless needed,
1452 * so it's safe to call this multiple times.
1453 *
1454 * This returns zero if unchanged, 1 if the configuration has changed,
1455 * or a negative error code.
1456 */
1457int snd_usb_endpoint_prepare(struct snd_usb_audio *chip,
1458			     struct snd_usb_endpoint *ep)
1459{
1460	bool iface_first;
1461	int err = 0;
1462
1463	mutex_lock(&chip->mutex);
1464	if (WARN_ON(!ep->iface_ref))
1465		goto unlock;
1466	if (!ep->need_prepare)
1467		goto unlock;
1468
1469	/* If the interface has been already set up, just set EP parameters */
1470	if (!ep->iface_ref->need_setup) {
1471		/* sample rate setup of UAC1 is per endpoint, and we need
1472		 * to update at each EP configuration
1473		 */
1474		if (ep->cur_audiofmt->protocol == UAC_VERSION_1) {
1475			err = init_sample_rate(chip, ep);
1476			if (err < 0)
1477				goto unlock;
1478		}
1479		goto done;
1480	}
1481
1482	/* Need to deselect altsetting at first */
1483	endpoint_set_interface(chip, ep, false);
1484
1485	/* Some UAC1 devices (e.g. Yamaha THR10) need the host interface
1486	 * to be set up before parameter setups
1487	 */
1488	iface_first = ep->cur_audiofmt->protocol == UAC_VERSION_1;
1489	/* Workaround for devices that require the interface setup at first like UAC1 */
1490	if (chip->quirk_flags & QUIRK_FLAG_SET_IFACE_FIRST)
1491		iface_first = true;
1492	if (iface_first) {
1493		err = endpoint_set_interface(chip, ep, true);
1494		if (err < 0)
1495			goto unlock;
1496	}
1497
1498	err = snd_usb_init_pitch(chip, ep->cur_audiofmt);
1499	if (err < 0)
1500		goto unlock;
1501
1502	err = init_sample_rate(chip, ep);
1503	if (err < 0)
1504		goto unlock;
1505
1506	err = snd_usb_select_mode_quirk(chip, ep->cur_audiofmt);
1507	if (err < 0)
1508		goto unlock;
1509
1510	/* for UAC2/3, enable the interface altset here at last */
1511	if (!iface_first) {
1512		err = endpoint_set_interface(chip, ep, true);
1513		if (err < 0)
1514			goto unlock;
1515	}
1516
1517	ep->iface_ref->need_setup = false;
1518
1519 done:
1520	ep->need_prepare = false;
1521	err = 1;
1522
1523unlock:
1524	mutex_unlock(&chip->mutex);
1525	return err;
1526}
1527
1528/* get the current rate set to the given clock by any endpoint */
1529int snd_usb_endpoint_get_clock_rate(struct snd_usb_audio *chip, int clock)
1530{
1531	struct snd_usb_clock_ref *ref;
1532	int rate = 0;
1533
1534	if (!clock)
1535		return 0;
1536	mutex_lock(&chip->mutex);
1537	list_for_each_entry(ref, &chip->clock_ref_list, list) {
1538		if (ref->clock == clock) {
1539			rate = ref->rate;
1540			break;
1541		}
1542	}
1543	mutex_unlock(&chip->mutex);
1544	return rate;
1545}
1546
1547/**
1548 * snd_usb_endpoint_start: start an snd_usb_endpoint
1549 *
1550 * @ep: the endpoint to start
1551 *
1552 * A call to this function will increment the running count of the endpoint.
1553 * In case it is not already running, the URBs for this endpoint will be
1554 * submitted. Otherwise, this function does nothing.
1555 *
1556 * Must be balanced to calls of snd_usb_endpoint_stop().
1557 *
1558 * Returns an error if the URB submission failed, 0 in all other cases.
1559 */
1560int snd_usb_endpoint_start(struct snd_usb_endpoint *ep)
1561{
1562	bool is_playback = usb_pipeout(ep->pipe);
1563	int err;
1564	unsigned int i;
1565
1566	if (atomic_read(&ep->chip->shutdown))
1567		return -EBADFD;
1568
1569	if (ep->sync_source)
1570		WRITE_ONCE(ep->sync_source->sync_sink, ep);
1571
1572	usb_audio_dbg(ep->chip, "Starting %s EP 0x%x (running %d)\n",
1573		      ep_type_name(ep->type), ep->ep_num,
1574		      atomic_read(&ep->running));
1575
1576	/* already running? */
1577	if (atomic_inc_return(&ep->running) != 1)
1578		return 0;
1579
1580	if (ep->clock_ref)
1581		atomic_inc(&ep->clock_ref->locked);
1582
1583	ep->active_mask = 0;
1584	ep->unlink_mask = 0;
1585	ep->phase = 0;
1586	ep->sample_accum = 0;
1587
1588	snd_usb_endpoint_start_quirk(ep);
1589
1590	/*
1591	 * If this endpoint has a data endpoint as implicit feedback source,
1592	 * don't start the urbs here. Instead, mark them all as available,
1593	 * wait for the record urbs to return and queue the playback urbs
1594	 * from that context.
1595	 */
1596
1597	if (!ep_state_update(ep, EP_STATE_STOPPED, EP_STATE_RUNNING))
1598		goto __error;
1599
1600	if (snd_usb_endpoint_implicit_feedback_sink(ep) &&
1601	    !(ep->chip->quirk_flags & QUIRK_FLAG_PLAYBACK_FIRST)) {
1602		usb_audio_dbg(ep->chip, "No URB submission due to implicit fb sync\n");
1603		i = 0;
1604		goto fill_rest;
1605	}
1606
1607	for (i = 0; i < ep->nurbs; i++) {
1608		struct urb *urb = ep->urb[i].urb;
1609
1610		if (snd_BUG_ON(!urb))
1611			goto __error;
1612
1613		if (is_playback)
1614			err = prepare_outbound_urb(ep, urb->context, true);
1615		else
1616			err = prepare_inbound_urb(ep, urb->context);
1617		if (err < 0) {
1618			/* stop filling at applptr */
1619			if (err == -EAGAIN)
1620				break;
1621			usb_audio_dbg(ep->chip,
1622				      "EP 0x%x: failed to prepare urb: %d\n",
1623				      ep->ep_num, err);
1624			goto __error;
1625		}
1626
1627		if (!atomic_read(&ep->chip->shutdown))
1628			err = usb_submit_urb(urb, GFP_ATOMIC);
1629		else
1630			err = -ENODEV;
1631		if (err < 0) {
1632			if (!atomic_read(&ep->chip->shutdown))
1633				usb_audio_err(ep->chip,
1634					      "cannot submit urb %d, error %d: %s\n",
1635					      i, err, usb_error_string(err));
1636			goto __error;
1637		}
1638		set_bit(i, &ep->active_mask);
1639		atomic_inc(&ep->submitted_urbs);
1640	}
1641
1642	if (!i) {
1643		usb_audio_dbg(ep->chip, "XRUN at starting EP 0x%x\n",
1644			      ep->ep_num);
1645		goto __error;
1646	}
1647
1648	usb_audio_dbg(ep->chip, "%d URBs submitted for EP 0x%x\n",
1649		      i, ep->ep_num);
1650
1651 fill_rest:
1652	/* put the remaining URBs to ready list */
1653	if (is_playback) {
1654		for (; i < ep->nurbs; i++)
1655			push_back_to_ready_list(ep, ep->urb + i);
1656	}
1657
1658	return 0;
1659
1660__error:
1661	snd_usb_endpoint_stop(ep, false);
1662	return -EPIPE;
1663}
1664
1665/**
1666 * snd_usb_endpoint_stop: stop an snd_usb_endpoint
1667 *
1668 * @ep: the endpoint to stop (may be NULL)
1669 * @keep_pending: keep in-flight URBs
1670 *
1671 * A call to this function will decrement the running count of the endpoint.
1672 * In case the last user has requested the endpoint stop, the URBs will
1673 * actually be deactivated.
1674 *
1675 * Must be balanced to calls of snd_usb_endpoint_start().
1676 *
1677 * The caller needs to synchronize the pending stop operation via
1678 * snd_usb_endpoint_sync_pending_stop().
1679 */
1680void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep, bool keep_pending)
1681{
1682	if (!ep)
1683		return;
1684
1685	usb_audio_dbg(ep->chip, "Stopping %s EP 0x%x (running %d)\n",
1686		      ep_type_name(ep->type), ep->ep_num,
1687		      atomic_read(&ep->running));
1688
1689	if (snd_BUG_ON(!atomic_read(&ep->running)))
1690		return;
1691
1692	if (!atomic_dec_return(&ep->running)) {
1693		if (ep->sync_source)
1694			WRITE_ONCE(ep->sync_source->sync_sink, NULL);
1695		stop_urbs(ep, false, keep_pending);
1696		if (ep->clock_ref)
1697			atomic_dec(&ep->clock_ref->locked);
1698
1699		if (ep->chip->quirk_flags & QUIRK_FLAG_FORCE_IFACE_RESET &&
1700		    usb_pipeout(ep->pipe)) {
1701			ep->need_prepare = true;
1702			if (ep->iface_ref)
1703				ep->iface_ref->need_setup = true;
1704		}
1705	}
1706}
1707
1708/**
1709 * snd_usb_endpoint_release: Tear down an snd_usb_endpoint
1710 *
1711 * @ep: the endpoint to release
1712 *
1713 * This function does not care for the endpoint's running count but will tear
1714 * down all the streaming URBs immediately.
1715 */
1716void snd_usb_endpoint_release(struct snd_usb_endpoint *ep)
1717{
1718	release_urbs(ep, true);
1719}
1720
1721/**
1722 * snd_usb_endpoint_free_all: Free the resources of an snd_usb_endpoint
1723 * @chip: The chip
1724 *
1725 * This free all endpoints and those resources
1726 */
1727void snd_usb_endpoint_free_all(struct snd_usb_audio *chip)
1728{
1729	struct snd_usb_endpoint *ep, *en;
1730	struct snd_usb_iface_ref *ip, *in;
1731	struct snd_usb_clock_ref *cp, *cn;
1732
1733	list_for_each_entry_safe(ep, en, &chip->ep_list, list)
1734		kfree(ep);
1735
1736	list_for_each_entry_safe(ip, in, &chip->iface_ref_list, list)
1737		kfree(ip);
1738
1739	list_for_each_entry_safe(cp, cn, &chip->clock_ref_list, list)
1740		kfree(cp);
1741}
1742
1743/*
1744 * snd_usb_handle_sync_urb: parse an USB sync packet
1745 *
1746 * @ep: the endpoint to handle the packet
1747 * @sender: the sending endpoint
1748 * @urb: the received packet
1749 *
1750 * This function is called from the context of an endpoint that received
1751 * the packet and is used to let another endpoint object handle the payload.
1752 */
1753static void snd_usb_handle_sync_urb(struct snd_usb_endpoint *ep,
1754				    struct snd_usb_endpoint *sender,
1755				    const struct urb *urb)
1756{
1757	int shift;
1758	unsigned int f;
1759	unsigned long flags;
1760
1761	snd_BUG_ON(ep == sender);
1762
1763	/*
1764	 * In case the endpoint is operating in implicit feedback mode, prepare
1765	 * a new outbound URB that has the same layout as the received packet
1766	 * and add it to the list of pending urbs. queue_pending_output_urbs()
1767	 * will take care of them later.
1768	 */
1769	if (snd_usb_endpoint_implicit_feedback_sink(ep) &&
1770	    atomic_read(&ep->running)) {
1771
1772		/* implicit feedback case */
1773		int i, bytes = 0;
1774		struct snd_urb_ctx *in_ctx;
1775		struct snd_usb_packet_info *out_packet;
1776
1777		in_ctx = urb->context;
1778
1779		/* Count overall packet size */
1780		for (i = 0; i < in_ctx->packets; i++)
1781			if (urb->iso_frame_desc[i].status == 0)
1782				bytes += urb->iso_frame_desc[i].actual_length;
1783
1784		/*
1785		 * skip empty packets. At least M-Audio's Fast Track Ultra stops
1786		 * streaming once it received a 0-byte OUT URB
1787		 */
1788		if (bytes == 0)
1789			return;
1790
1791		spin_lock_irqsave(&ep->lock, flags);
1792		if (ep->next_packet_queued >= ARRAY_SIZE(ep->next_packet)) {
1793			spin_unlock_irqrestore(&ep->lock, flags);
1794			usb_audio_err(ep->chip,
1795				      "next package FIFO overflow EP 0x%x\n",
1796				      ep->ep_num);
1797			notify_xrun(ep);
1798			return;
1799		}
1800
1801		out_packet = next_packet_fifo_enqueue(ep);
1802
1803		/*
1804		 * Iterate through the inbound packet and prepare the lengths
1805		 * for the output packet. The OUT packet we are about to send
1806		 * will have the same amount of payload bytes per stride as the
1807		 * IN packet we just received. Since the actual size is scaled
1808		 * by the stride, use the sender stride to calculate the length
1809		 * in case the number of channels differ between the implicitly
1810		 * fed-back endpoint and the synchronizing endpoint.
1811		 */
1812
1813		out_packet->packets = in_ctx->packets;
1814		for (i = 0; i < in_ctx->packets; i++) {
1815			if (urb->iso_frame_desc[i].status == 0)
1816				out_packet->packet_size[i] =
1817					urb->iso_frame_desc[i].actual_length / sender->stride;
1818			else
1819				out_packet->packet_size[i] = 0;
1820		}
1821
1822		spin_unlock_irqrestore(&ep->lock, flags);
1823		snd_usb_queue_pending_output_urbs(ep, false);
1824
1825		return;
1826	}
1827
1828	/*
1829	 * process after playback sync complete
1830	 *
1831	 * Full speed devices report feedback values in 10.14 format as samples
1832	 * per frame, high speed devices in 16.16 format as samples per
1833	 * microframe.
1834	 *
1835	 * Because the Audio Class 1 spec was written before USB 2.0, many high
1836	 * speed devices use a wrong interpretation, some others use an
1837	 * entirely different format.
1838	 *
1839	 * Therefore, we cannot predict what format any particular device uses
1840	 * and must detect it automatically.
1841	 */
1842
1843	if (urb->iso_frame_desc[0].status != 0 ||
1844	    urb->iso_frame_desc[0].actual_length < 3)
1845		return;
1846
1847	f = le32_to_cpup(urb->transfer_buffer);
1848	if (urb->iso_frame_desc[0].actual_length == 3)
1849		f &= 0x00ffffff;
1850	else
1851		f &= 0x0fffffff;
1852
1853	if (f == 0)
1854		return;
1855
1856	if (unlikely(sender->tenor_fb_quirk)) {
1857		/*
1858		 * Devices based on Tenor 8802 chipsets (TEAC UD-H01
1859		 * and others) sometimes change the feedback value
1860		 * by +/- 0x1.0000.
1861		 */
1862		if (f < ep->freqn - 0x8000)
1863			f += 0xf000;
1864		else if (f > ep->freqn + 0x8000)
1865			f -= 0xf000;
1866	} else if (unlikely(ep->freqshift == INT_MIN)) {
1867		/*
1868		 * The first time we see a feedback value, determine its format
1869		 * by shifting it left or right until it matches the nominal
1870		 * frequency value.  This assumes that the feedback does not
1871		 * differ from the nominal value more than +50% or -25%.
1872		 */
1873		shift = 0;
1874		while (f < ep->freqn - ep->freqn / 4) {
1875			f <<= 1;
1876			shift++;
1877		}
1878		while (f > ep->freqn + ep->freqn / 2) {
1879			f >>= 1;
1880			shift--;
1881		}
1882		ep->freqshift = shift;
1883	} else if (ep->freqshift >= 0)
1884		f <<= ep->freqshift;
1885	else
1886		f >>= -ep->freqshift;
1887
1888	if (likely(f >= ep->freqn - ep->freqn / 8 && f <= ep->freqmax)) {
1889		/*
1890		 * If the frequency looks valid, set it.
1891		 * This value is referred to in prepare_playback_urb().
1892		 */
1893		spin_lock_irqsave(&ep->lock, flags);
1894		ep->freqm = f;
1895		spin_unlock_irqrestore(&ep->lock, flags);
1896	} else {
1897		/*
1898		 * Out of range; maybe the shift value is wrong.
1899		 * Reset it so that we autodetect again the next time.
1900		 */
1901		ep->freqshift = INT_MIN;
1902	}
1903}
1904