1// SPDX-License-Identifier: GPL-2.0
   2
   3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
   4 * Copyright (C) 2019-2023 Linaro Ltd.
   5 */
   6
   7#include <linux/types.h>
   8#include <linux/device.h>
   9#include <linux/slab.h>
  10#include <linux/bitfield.h>
  11#include <linux/if_rmnet.h>
 
  12#include <linux/dma-direction.h>
 
 
  13
  14#include "gsi.h"
  15#include "gsi_trans.h"
  16#include "ipa.h"
 
  17#include "ipa_data.h"
  18#include "ipa_endpoint.h"
  19#include "ipa_cmd.h"
 
  20#include "ipa_mem.h"
  21#include "ipa_modem.h"
  22#include "ipa_table.h"
  23#include "ipa_gsi.h"
  24#include "ipa_power.h"
 
 
 
  25
  26/* Hardware is told about receive buffers once a "batch" has been queued */
  27#define IPA_REPLENISH_BATCH	16		/* Must be non-zero */
  28
  29/* The amount of RX buffer space consumed by standard skb overhead */
  30#define IPA_RX_BUFFER_OVERHEAD	(PAGE_SIZE - SKB_MAX_ORDER(NET_SKB_PAD, 0))
  31
  32/* Where to find the QMAP mux_id for a packet within modem-supplied metadata */
  33#define IPA_ENDPOINT_QMAP_METADATA_MASK		0x000000ff /* host byte order */
  34
  35#define IPA_ENDPOINT_RESET_AGGR_RETRY_MAX	3
  36
  37/** enum ipa_status_opcode - IPA status opcode field hardware values */
  38enum ipa_status_opcode {				/* *Not* a bitmask */
  39	IPA_STATUS_OPCODE_PACKET		= 1,
  40	IPA_STATUS_OPCODE_NEW_RULE_PACKET	= 2,
  41	IPA_STATUS_OPCODE_DROPPED_PACKET	= 4,
  42	IPA_STATUS_OPCODE_SUSPENDED_PACKET	= 8,
  43	IPA_STATUS_OPCODE_LOG			= 16,
  44	IPA_STATUS_OPCODE_DCMP			= 32,
  45	IPA_STATUS_OPCODE_PACKET_2ND_PASS	= 64,
  46};
  47
  48/** enum ipa_status_exception - IPA status exception field hardware values */
  49enum ipa_status_exception {				/* *Not* a bitmask */
  50	/* 0 means no exception */
  51	IPA_STATUS_EXCEPTION_DEAGGR		= 1,
  52	IPA_STATUS_EXCEPTION_IPTYPE		= 4,
  53	IPA_STATUS_EXCEPTION_PACKET_LENGTH	= 8,
  54	IPA_STATUS_EXCEPTION_FRAG_RULE_MISS	= 16,
  55	IPA_STATUS_EXCEPTION_SW_FILTER		= 32,
  56	IPA_STATUS_EXCEPTION_NAT		= 64,		/* IPv4 */
  57	IPA_STATUS_EXCEPTION_IPV6_CONN_TRACK	= 64,		/* IPv6 */
  58	IPA_STATUS_EXCEPTION_UC			= 128,
  59	IPA_STATUS_EXCEPTION_INVALID_ENDPOINT	= 129,
  60	IPA_STATUS_EXCEPTION_HEADER_INSERT	= 136,
  61	IPA_STATUS_EXCEPTION_CHEKCSUM		= 229,
  62};
  63
  64/** enum ipa_status_mask - IPA status mask field bitmask hardware values */
  65enum ipa_status_mask {
  66	IPA_STATUS_MASK_FRAG_PROCESS		= BIT(0),
  67	IPA_STATUS_MASK_FILT_PROCESS		= BIT(1),
  68	IPA_STATUS_MASK_NAT_PROCESS		= BIT(2),
  69	IPA_STATUS_MASK_ROUTE_PROCESS		= BIT(3),
  70	IPA_STATUS_MASK_TAG_VALID		= BIT(4),
  71	IPA_STATUS_MASK_FRAGMENT		= BIT(5),
  72	IPA_STATUS_MASK_FIRST_FRAGMENT		= BIT(6),
  73	IPA_STATUS_MASK_V4			= BIT(7),
  74	IPA_STATUS_MASK_CKSUM_PROCESS		= BIT(8),
  75	IPA_STATUS_MASK_AGGR_PROCESS		= BIT(9),
  76	IPA_STATUS_MASK_DEST_EOT		= BIT(10),
  77	IPA_STATUS_MASK_DEAGGR_PROCESS		= BIT(11),
  78	IPA_STATUS_MASK_DEAGG_FIRST		= BIT(12),
  79	IPA_STATUS_MASK_SRC_EOT			= BIT(13),
  80	IPA_STATUS_MASK_PREV_EOT		= BIT(14),
  81	IPA_STATUS_MASK_BYTE_LIMIT		= BIT(15),
  82};
  83
  84/* Special IPA filter/router rule field value indicating "rule miss" */
  85#define IPA_STATUS_RULE_MISS	0x3ff	/* 10-bit filter/router rule fields */
  86
  87/** The IPA status nat_type field uses enum ipa_nat_type hardware values */
  88
  89/* enum ipa_status_field_id - IPA packet status structure field identifiers */
  90enum ipa_status_field_id {
  91	STATUS_OPCODE,			/* enum ipa_status_opcode */
  92	STATUS_EXCEPTION,		/* enum ipa_status_exception */
  93	STATUS_MASK,			/* enum ipa_status_mask (bitmask) */
  94	STATUS_LENGTH,
  95	STATUS_SRC_ENDPOINT,
  96	STATUS_DST_ENDPOINT,
  97	STATUS_METADATA,
  98	STATUS_FILTER_LOCAL,		/* Boolean */
  99	STATUS_FILTER_HASH,		/* Boolean */
 100	STATUS_FILTER_GLOBAL,		/* Boolean */
 101	STATUS_FILTER_RETAIN,		/* Boolean */
 102	STATUS_FILTER_RULE_INDEX,
 103	STATUS_ROUTER_LOCAL,		/* Boolean */
 104	STATUS_ROUTER_HASH,		/* Boolean */
 105	STATUS_UCP,			/* Boolean */
 106	STATUS_ROUTER_TABLE,
 107	STATUS_ROUTER_RULE_INDEX,
 108	STATUS_NAT_HIT,			/* Boolean */
 109	STATUS_NAT_INDEX,
 110	STATUS_NAT_TYPE,		/* enum ipa_nat_type */
 111	STATUS_TAG_LOW32,		/* Low-order 32 bits of 48-bit tag */
 112	STATUS_TAG_HIGH16,		/* High-order 16 bits of 48-bit tag */
 113	STATUS_SEQUENCE,
 114	STATUS_TIME_OF_DAY,
 115	STATUS_HEADER_LOCAL,		/* Boolean */
 116	STATUS_HEADER_OFFSET,
 117	STATUS_FRAG_HIT,		/* Boolean */
 118	STATUS_FRAG_RULE_INDEX,
 119};
 120
 121/* Size in bytes of an IPA packet status structure */
 122#define IPA_STATUS_SIZE			sizeof(__le32[8])
 123
 124/* IPA status structure decoder; looks up field values for a structure */
 125static u32 ipa_status_extract(struct ipa *ipa, const void *data,
 126			      enum ipa_status_field_id field)
 127{
 128	enum ipa_version version = ipa->version;
 129	const __le32 *word = data;
 130
 131	switch (field) {
 132	case STATUS_OPCODE:
 133		return le32_get_bits(word[0], GENMASK(7, 0));
 134	case STATUS_EXCEPTION:
 135		return le32_get_bits(word[0], GENMASK(15, 8));
 136	case STATUS_MASK:
 137		return le32_get_bits(word[0], GENMASK(31, 16));
 138	case STATUS_LENGTH:
 139		return le32_get_bits(word[1], GENMASK(15, 0));
 140	case STATUS_SRC_ENDPOINT:
 141		if (version < IPA_VERSION_5_0)
 142			return le32_get_bits(word[1], GENMASK(20, 16));
 143		return le32_get_bits(word[1], GENMASK(23, 16));
 144	/* Status word 1, bits 21-23 are reserved (not IPA v5.0+) */
 145	/* Status word 1, bits 24-26 are reserved (IPA v5.0+) */
 146	case STATUS_DST_ENDPOINT:
 147		if (version < IPA_VERSION_5_0)
 148			return le32_get_bits(word[1], GENMASK(28, 24));
 149		return le32_get_bits(word[7], GENMASK(23, 16));
 150	/* Status word 1, bits 29-31 are reserved */
 151	case STATUS_METADATA:
 152		return le32_to_cpu(word[2]);
 153	case STATUS_FILTER_LOCAL:
 154		return le32_get_bits(word[3], GENMASK(0, 0));
 155	case STATUS_FILTER_HASH:
 156		return le32_get_bits(word[3], GENMASK(1, 1));
 157	case STATUS_FILTER_GLOBAL:
 158		return le32_get_bits(word[3], GENMASK(2, 2));
 159	case STATUS_FILTER_RETAIN:
 160		return le32_get_bits(word[3], GENMASK(3, 3));
 161	case STATUS_FILTER_RULE_INDEX:
 162		return le32_get_bits(word[3], GENMASK(13, 4));
 163	/* ROUTER_TABLE is in word 3, bits 14-21 (IPA v5.0+) */
 164	case STATUS_ROUTER_LOCAL:
 165		if (version < IPA_VERSION_5_0)
 166			return le32_get_bits(word[3], GENMASK(14, 14));
 167		return le32_get_bits(word[1], GENMASK(27, 27));
 168	case STATUS_ROUTER_HASH:
 169		if (version < IPA_VERSION_5_0)
 170			return le32_get_bits(word[3], GENMASK(15, 15));
 171		return le32_get_bits(word[1], GENMASK(28, 28));
 172	case STATUS_UCP:
 173		if (version < IPA_VERSION_5_0)
 174			return le32_get_bits(word[3], GENMASK(16, 16));
 175		return le32_get_bits(word[7], GENMASK(31, 31));
 176	case STATUS_ROUTER_TABLE:
 177		if (version < IPA_VERSION_5_0)
 178			return le32_get_bits(word[3], GENMASK(21, 17));
 179		return le32_get_bits(word[3], GENMASK(21, 14));
 180	case STATUS_ROUTER_RULE_INDEX:
 181		return le32_get_bits(word[3], GENMASK(31, 22));
 182	case STATUS_NAT_HIT:
 183		return le32_get_bits(word[4], GENMASK(0, 0));
 184	case STATUS_NAT_INDEX:
 185		return le32_get_bits(word[4], GENMASK(13, 1));
 186	case STATUS_NAT_TYPE:
 187		return le32_get_bits(word[4], GENMASK(15, 14));
 188	case STATUS_TAG_LOW32:
 189		return le32_get_bits(word[4], GENMASK(31, 16)) |
 190			(le32_get_bits(word[5], GENMASK(15, 0)) << 16);
 191	case STATUS_TAG_HIGH16:
 192		return le32_get_bits(word[5], GENMASK(31, 16));
 193	case STATUS_SEQUENCE:
 194		return le32_get_bits(word[6], GENMASK(7, 0));
 195	case STATUS_TIME_OF_DAY:
 196		return le32_get_bits(word[6], GENMASK(31, 8));
 197	case STATUS_HEADER_LOCAL:
 198		return le32_get_bits(word[7], GENMASK(0, 0));
 199	case STATUS_HEADER_OFFSET:
 200		return le32_get_bits(word[7], GENMASK(10, 1));
 201	case STATUS_FRAG_HIT:
 202		return le32_get_bits(word[7], GENMASK(11, 11));
 203	case STATUS_FRAG_RULE_INDEX:
 204		return le32_get_bits(word[7], GENMASK(15, 12));
 205	/* Status word 7, bits 16-30 are reserved */
 206	/* Status word 7, bit 31 is reserved (not IPA v5.0+) */
 207	default:
 208		WARN(true, "%s: bad field_id %u\n", __func__, field);
 209		return 0;
 210	}
 211}
 212
 213/* Compute the aggregation size value to use for a given buffer size */
 214static u32 ipa_aggr_size_kb(u32 rx_buffer_size, bool aggr_hard_limit)
 215{
 216	/* A hard aggregation limit will not be crossed; aggregation closes
 217	 * if saving incoming data would cross the hard byte limit boundary.
 218	 *
 219	 * With a soft limit, aggregation closes *after* the size boundary
 220	 * has been crossed.  In that case the limit must leave enough space
 221	 * after that limit to receive a full MTU of data plus overhead.
 222	 */
 223	if (!aggr_hard_limit)
 224		rx_buffer_size -= IPA_MTU + IPA_RX_BUFFER_OVERHEAD;
 225
 226	/* The byte limit is encoded as a number of kilobytes */
 227
 228	return rx_buffer_size / SZ_1K;
 229}
 230
 231static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count,
 232			    const struct ipa_gsi_endpoint_data *all_data,
 233			    const struct ipa_gsi_endpoint_data *data)
 234{
 235	const struct ipa_gsi_endpoint_data *other_data;
 236	struct device *dev = &ipa->pdev->dev;
 237	enum ipa_endpoint_name other_name;
 
 238
 239	if (ipa_gsi_endpoint_data_empty(data))
 240		return true;
 241
 242	if (!data->toward_ipa) {
 243		const struct ipa_endpoint_rx *rx_config;
 244		const struct reg *reg;
 245		u32 buffer_size;
 246		u32 aggr_size;
 247		u32 limit;
 248
 249		if (data->endpoint.filter_support) {
 250			dev_err(dev, "filtering not supported for "
 251					"RX endpoint %u\n",
 252				data->endpoint_id);
 253			return false;
 254		}
 255
 256		/* Nothing more to check for non-AP RX */
 257		if (data->ee_id != GSI_EE_AP)
 258			return true;
 259
 260		rx_config = &data->endpoint.config.rx;
 261
 262		/* The buffer size must hold an MTU plus overhead */
 263		buffer_size = rx_config->buffer_size;
 264		limit = IPA_MTU + IPA_RX_BUFFER_OVERHEAD;
 265		if (buffer_size < limit) {
 266			dev_err(dev, "RX buffer size too small for RX endpoint %u (%u < %u)\n",
 267				data->endpoint_id, buffer_size, limit);
 268			return false;
 269		}
 270
 271		if (!data->endpoint.config.aggregation) {
 272			bool result = true;
 273
 274			/* No aggregation; check for bogus aggregation data */
 275			if (rx_config->aggr_time_limit) {
 276				dev_err(dev,
 277					"time limit with no aggregation for RX endpoint %u\n",
 278					data->endpoint_id);
 279				result = false;
 280			}
 281
 282			if (rx_config->aggr_hard_limit) {
 283				dev_err(dev, "hard limit with no aggregation for RX endpoint %u\n",
 284					data->endpoint_id);
 285				result = false;
 286			}
 287
 288			if (rx_config->aggr_close_eof) {
 289				dev_err(dev, "close EOF with no aggregation for RX endpoint %u\n",
 290					data->endpoint_id);
 291				result = false;
 292			}
 293
 294			return result;	/* Nothing more to check */
 295		}
 296
 297		/* For an endpoint supporting receive aggregation, the byte
 298		 * limit defines the point at which aggregation closes.  This
 299		 * check ensures the receive buffer size doesn't result in a
 300		 * limit that exceeds what's representable in the aggregation
 301		 * byte limit field.
 302		 */
 303		aggr_size = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD,
 304					     rx_config->aggr_hard_limit);
 305		reg = ipa_reg(ipa, ENDP_INIT_AGGR);
 306
 307		limit = reg_field_max(reg, BYTE_LIMIT);
 308		if (aggr_size > limit) {
 309			dev_err(dev, "aggregated size too large for RX endpoint %u (%u KB > %u KB)\n",
 310				data->endpoint_id, aggr_size, limit);
 311
 312			return false;
 313		}
 314
 315		return true;	/* Nothing more to check for RX */
 316	}
 317
 318	/* Starting with IPA v4.5 sequencer replication is obsolete */
 319	if (ipa->version >= IPA_VERSION_4_5) {
 320		if (data->endpoint.config.tx.seq_rep_type) {
 321			dev_err(dev, "no-zero seq_rep_type TX endpoint %u\n",
 322				data->endpoint_id);
 323			return false;
 324		}
 325	}
 326
 327	if (data->endpoint.config.status_enable) {
 328		other_name = data->endpoint.config.tx.status_endpoint;
 329		if (other_name >= count) {
 330			dev_err(dev, "status endpoint name %u out of range "
 331					"for endpoint %u\n",
 332				other_name, data->endpoint_id);
 333			return false;
 334		}
 335
 336		/* Status endpoint must be defined... */
 337		other_data = &all_data[other_name];
 338		if (ipa_gsi_endpoint_data_empty(other_data)) {
 339			dev_err(dev, "DMA endpoint name %u undefined "
 340					"for endpoint %u\n",
 341				other_name, data->endpoint_id);
 342			return false;
 343		}
 344
 345		/* ...and has to be an RX endpoint... */
 346		if (other_data->toward_ipa) {
 347			dev_err(dev,
 348				"status endpoint for endpoint %u not RX\n",
 349				data->endpoint_id);
 350			return false;
 351		}
 352
 353		/* ...and if it's to be an AP endpoint... */
 354		if (other_data->ee_id == GSI_EE_AP) {
 355			/* ...make sure it has status enabled. */
 356			if (!other_data->endpoint.config.status_enable) {
 357				dev_err(dev,
 358					"status not enabled for endpoint %u\n",
 359					other_data->endpoint_id);
 360				return false;
 361			}
 362		}
 363	}
 364
 365	if (data->endpoint.config.dma_mode) {
 366		other_name = data->endpoint.config.dma_endpoint;
 367		if (other_name >= count) {
 368			dev_err(dev, "DMA endpoint name %u out of range "
 369					"for endpoint %u\n",
 370				other_name, data->endpoint_id);
 371			return false;
 372		}
 373
 374		other_data = &all_data[other_name];
 375		if (ipa_gsi_endpoint_data_empty(other_data)) {
 376			dev_err(dev, "DMA endpoint name %u undefined "
 377					"for endpoint %u\n",
 378				other_name, data->endpoint_id);
 379			return false;
 380		}
 381	}
 382
 383	return true;
 384}
 385
 386/* Validate endpoint configuration data.  Return max defined endpoint ID */
 387static u32 ipa_endpoint_max(struct ipa *ipa, u32 count,
 388			    const struct ipa_gsi_endpoint_data *data)
 389{
 390	const struct ipa_gsi_endpoint_data *dp = data;
 391	struct device *dev = &ipa->pdev->dev;
 392	enum ipa_endpoint_name name;
 393	u32 max;
 394
 395	if (count > IPA_ENDPOINT_COUNT) {
 396		dev_err(dev, "too many endpoints specified (%u > %u)\n",
 397			count, IPA_ENDPOINT_COUNT);
 398		return 0;
 399	}
 400
 401	/* Make sure needed endpoints have defined data */
 402	if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_COMMAND_TX])) {
 403		dev_err(dev, "command TX endpoint not defined\n");
 404		return 0;
 405	}
 406	if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_LAN_RX])) {
 407		dev_err(dev, "LAN RX endpoint not defined\n");
 408		return 0;
 409	}
 410	if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_MODEM_TX])) {
 411		dev_err(dev, "AP->modem TX endpoint not defined\n");
 412		return 0;
 413	}
 414	if (ipa_gsi_endpoint_data_empty(&data[IPA_ENDPOINT_AP_MODEM_RX])) {
 415		dev_err(dev, "AP<-modem RX endpoint not defined\n");
 416		return 0;
 417	}
 418
 419	max = 0;
 420	for (name = 0; name < count; name++, dp++) {
 421		if (!ipa_endpoint_data_valid_one(ipa, count, data, dp))
 422			return 0;
 423		max = max_t(u32, max, dp->endpoint_id);
 424	}
 425
 426	return max;
 427}
 428
 429/* Allocate a transaction to use on a non-command endpoint */
 430static struct gsi_trans *ipa_endpoint_trans_alloc(struct ipa_endpoint *endpoint,
 431						  u32 tre_count)
 432{
 433	struct gsi *gsi = &endpoint->ipa->gsi;
 434	u32 channel_id = endpoint->channel_id;
 435	enum dma_data_direction direction;
 436
 437	direction = endpoint->toward_ipa ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 438
 439	return gsi_channel_trans_alloc(gsi, channel_id, tre_count, direction);
 440}
 441
 442/* suspend_delay represents suspend for RX, delay for TX endpoints.
 443 * Note that suspend is not supported starting with IPA v4.0, and
 444 * delay mode should not be used starting with IPA v4.2.
 445 */
 446static bool
 447ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay)
 448{
 449	struct ipa *ipa = endpoint->ipa;
 450	const struct reg *reg;
 451	u32 field_id;
 452	u32 offset;
 453	bool state;
 454	u32 mask;
 455	u32 val;
 456
 457	if (endpoint->toward_ipa)
 458		WARN_ON(ipa->version >= IPA_VERSION_4_2);
 459	else
 460		WARN_ON(ipa->version >= IPA_VERSION_4_0);
 461
 462	reg = ipa_reg(ipa, ENDP_INIT_CTRL);
 463	offset = reg_n_offset(reg, endpoint->endpoint_id);
 464	val = ioread32(ipa->reg_virt + offset);
 465
 466	field_id = endpoint->toward_ipa ? ENDP_DELAY : ENDP_SUSPEND;
 467	mask = reg_bit(reg, field_id);
 468
 469	state = !!(val & mask);
 470
 471	/* Don't bother if it's already in the requested state */
 472	if (suspend_delay != state) {
 473		val ^= mask;
 474		iowrite32(val, ipa->reg_virt + offset);
 475	}
 476
 477	return state;
 478}
 479
 480/* We don't care what the previous state was for delay mode */
 481static void
 482ipa_endpoint_program_delay(struct ipa_endpoint *endpoint, bool enable)
 483{
 484	/* Delay mode should not be used for IPA v4.2+ */
 485	WARN_ON(endpoint->ipa->version >= IPA_VERSION_4_2);
 486	WARN_ON(!endpoint->toward_ipa);
 487
 488	(void)ipa_endpoint_init_ctrl(endpoint, enable);
 489}
 490
 491static bool ipa_endpoint_aggr_active(struct ipa_endpoint *endpoint)
 492{
 493	u32 endpoint_id = endpoint->endpoint_id;
 494	struct ipa *ipa = endpoint->ipa;
 495	u32 unit = endpoint_id / 32;
 496	const struct reg *reg;
 497	u32 val;
 498
 499	WARN_ON(!test_bit(endpoint_id, ipa->available));
 500
 501	reg = ipa_reg(ipa, STATE_AGGR_ACTIVE);
 502	val = ioread32(ipa->reg_virt + reg_n_offset(reg, unit));
 503
 504	return !!(val & BIT(endpoint_id % 32));
 505}
 506
 507static void ipa_endpoint_force_close(struct ipa_endpoint *endpoint)
 508{
 509	u32 endpoint_id = endpoint->endpoint_id;
 510	u32 mask = BIT(endpoint_id % 32);
 511	struct ipa *ipa = endpoint->ipa;
 512	u32 unit = endpoint_id / 32;
 513	const struct reg *reg;
 514
 515	WARN_ON(!test_bit(endpoint_id, ipa->available));
 516
 517	reg = ipa_reg(ipa, AGGR_FORCE_CLOSE);
 518	iowrite32(mask, ipa->reg_virt + reg_n_offset(reg, unit));
 519}
 520
 521/**
 522 * ipa_endpoint_suspend_aggr() - Emulate suspend interrupt
 523 * @endpoint:	Endpoint on which to emulate a suspend
 524 *
 525 *  Emulate suspend IPA interrupt to unsuspend an endpoint suspended
 526 *  with an open aggregation frame.  This is to work around a hardware
 527 *  issue in IPA version 3.5.1 where the suspend interrupt will not be
 528 *  generated when it should be.
 529 */
 530static void ipa_endpoint_suspend_aggr(struct ipa_endpoint *endpoint)
 531{
 532	struct ipa *ipa = endpoint->ipa;
 533
 534	if (!endpoint->config.aggregation)
 535		return;
 536
 537	/* Nothing to do if the endpoint doesn't have aggregation open */
 538	if (!ipa_endpoint_aggr_active(endpoint))
 539		return;
 540
 541	/* Force close aggregation */
 542	ipa_endpoint_force_close(endpoint);
 543
 544	ipa_interrupt_simulate_suspend(ipa->interrupt);
 545}
 546
 547/* Returns previous suspend state (true means suspend was enabled) */
 548static bool
 549ipa_endpoint_program_suspend(struct ipa_endpoint *endpoint, bool enable)
 550{
 551	bool suspended;
 552
 553	if (endpoint->ipa->version >= IPA_VERSION_4_0)
 554		return enable;	/* For IPA v4.0+, no change made */
 555
 556	WARN_ON(endpoint->toward_ipa);
 557
 558	suspended = ipa_endpoint_init_ctrl(endpoint, enable);
 559
 560	/* A client suspended with an open aggregation frame will not
 561	 * generate a SUSPEND IPA interrupt.  If enabling suspend, have
 562	 * ipa_endpoint_suspend_aggr() handle this.
 563	 */
 564	if (enable && !suspended)
 565		ipa_endpoint_suspend_aggr(endpoint);
 566
 567	return suspended;
 568}
 569
 570/* Put all modem RX endpoints into suspend mode, and stop transmission
 571 * on all modem TX endpoints.  Prior to IPA v4.2, endpoint DELAY mode is
 572 * used for TX endpoints; starting with IPA v4.2 we use GSI channel flow
 573 * control instead.
 574 */
 575void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable)
 576{
 577	u32 endpoint_id = 0;
 578
 579	while (endpoint_id < ipa->endpoint_count) {
 580		struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++];
 581
 582		if (endpoint->ee_id != GSI_EE_MODEM)
 583			continue;
 584
 585		if (!endpoint->toward_ipa)
 586			(void)ipa_endpoint_program_suspend(endpoint, enable);
 587		else if (ipa->version < IPA_VERSION_4_2)
 588			ipa_endpoint_program_delay(endpoint, enable);
 589		else
 590			gsi_modem_channel_flow_control(&ipa->gsi,
 591						       endpoint->channel_id,
 592						       enable);
 593	}
 594}
 595
 596/* Reset all modem endpoints to use the default exception endpoint */
 597int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa)
 598{
 599	struct gsi_trans *trans;
 600	u32 endpoint_id;
 601	u32 count;
 602
 603	/* We need one command per modem TX endpoint, plus the commands
 604	 * that clear the pipeline.
 605	 */
 606	count = ipa->modem_tx_count + ipa_cmd_pipeline_clear_count();
 607	trans = ipa_cmd_trans_alloc(ipa, count);
 608	if (!trans) {
 609		dev_err(&ipa->pdev->dev,
 610			"no transaction to reset modem exception endpoints\n");
 611		return -EBUSY;
 612	}
 613
 614	for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) {
 615		struct ipa_endpoint *endpoint;
 616		const struct reg *reg;
 617		u32 offset;
 618
 619		/* We only reset modem TX endpoints */
 620		endpoint = &ipa->endpoint[endpoint_id];
 621		if (!(endpoint->ee_id == GSI_EE_MODEM && endpoint->toward_ipa))
 622			continue;
 623
 624		reg = ipa_reg(ipa, ENDP_STATUS);
 625		offset = reg_n_offset(reg, endpoint_id);
 626
 627		/* Value written is 0, and all bits are updated.  That
 628		 * means status is disabled on the endpoint, and as a
 629		 * result all other fields in the register are ignored.
 630		 */
 631		ipa_cmd_register_write_add(trans, offset, 0, ~0, false);
 632	}
 633
 634	ipa_cmd_pipeline_clear_add(trans);
 635
 636	gsi_trans_commit_wait(trans);
 637
 638	ipa_cmd_pipeline_clear_wait(ipa);
 639
 640	return 0;
 641}
 642
 643static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint)
 644{
 645	u32 endpoint_id = endpoint->endpoint_id;
 646	struct ipa *ipa = endpoint->ipa;
 647	enum ipa_cs_offload_en enabled;
 648	const struct reg *reg;
 649	u32 val = 0;
 650
 651	reg = ipa_reg(ipa, ENDP_INIT_CFG);
 652	/* FRAG_OFFLOAD_EN is 0 */
 653	if (endpoint->config.checksum) {
 654		enum ipa_version version = ipa->version;
 655
 656		if (endpoint->toward_ipa) {
 657			u32 off;
 658
 659			/* Checksum header offset is in 4-byte units */
 660			off = sizeof(struct rmnet_map_header) / sizeof(u32);
 661			val |= reg_encode(reg, CS_METADATA_HDR_OFFSET, off);
 662
 663			enabled = version < IPA_VERSION_4_5
 664					? IPA_CS_OFFLOAD_UL
 665					: IPA_CS_OFFLOAD_INLINE;
 666		} else {
 667			enabled = version < IPA_VERSION_4_5
 668					? IPA_CS_OFFLOAD_DL
 669					: IPA_CS_OFFLOAD_INLINE;
 670		}
 671	} else {
 672		enabled = IPA_CS_OFFLOAD_NONE;
 673	}
 674	val |= reg_encode(reg, CS_OFFLOAD_EN, enabled);
 675	/* CS_GEN_QMB_MASTER_SEL is 0 */
 676
 677	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
 678}
 679
 680static void ipa_endpoint_init_nat(struct ipa_endpoint *endpoint)
 681{
 682	u32 endpoint_id = endpoint->endpoint_id;
 683	struct ipa *ipa = endpoint->ipa;
 684	const struct reg *reg;
 685	u32 val;
 686
 687	if (!endpoint->toward_ipa)
 688		return;
 689
 690	reg = ipa_reg(ipa, ENDP_INIT_NAT);
 691	val = reg_encode(reg, NAT_EN, IPA_NAT_TYPE_BYPASS);
 692
 693	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
 694}
 695
 696static u32
 697ipa_qmap_header_size(enum ipa_version version, struct ipa_endpoint *endpoint)
 698{
 699	u32 header_size = sizeof(struct rmnet_map_header);
 700
 701	/* Without checksum offload, we just have the MAP header */
 702	if (!endpoint->config.checksum)
 703		return header_size;
 704
 705	if (version < IPA_VERSION_4_5) {
 706		/* Checksum header inserted for AP TX endpoints only */
 707		if (endpoint->toward_ipa)
 708			header_size += sizeof(struct rmnet_map_ul_csum_header);
 709	} else {
 710		/* Checksum header is used in both directions */
 711		header_size += sizeof(struct rmnet_map_v5_csum_header);
 712	}
 713
 714	return header_size;
 715}
 716
 717/* Encoded value for ENDP_INIT_HDR register HDR_LEN* field(s) */
 718static u32 ipa_header_size_encode(enum ipa_version version,
 719				  const struct reg *reg, u32 header_size)
 720{
 721	u32 field_max = reg_field_max(reg, HDR_LEN);
 722	u32 val;
 723
 724	/* We know field_max can be used as a mask (2^n - 1) */
 725	val = reg_encode(reg, HDR_LEN, header_size & field_max);
 726	if (version < IPA_VERSION_4_5) {
 727		WARN_ON(header_size > field_max);
 728		return val;
 729	}
 730
 731	/* IPA v4.5 adds a few more most-significant bits */
 732	header_size >>= hweight32(field_max);
 733	WARN_ON(header_size > reg_field_max(reg, HDR_LEN_MSB));
 734	val |= reg_encode(reg, HDR_LEN_MSB, header_size);
 735
 736	return val;
 737}
 738
 739/* Encoded value for ENDP_INIT_HDR register OFST_METADATA* field(s) */
 740static u32 ipa_metadata_offset_encode(enum ipa_version version,
 741				      const struct reg *reg, u32 offset)
 742{
 743	u32 field_max = reg_field_max(reg, HDR_OFST_METADATA);
 744	u32 val;
 745
 746	/* We know field_max can be used as a mask (2^n - 1) */
 747	val = reg_encode(reg, HDR_OFST_METADATA, offset);
 748	if (version < IPA_VERSION_4_5) {
 749		WARN_ON(offset > field_max);
 750		return val;
 751	}
 752
 753	/* IPA v4.5 adds a few more most-significant bits */
 754	offset >>= hweight32(field_max);
 755	WARN_ON(offset > reg_field_max(reg, HDR_OFST_METADATA_MSB));
 756	val |= reg_encode(reg, HDR_OFST_METADATA_MSB, offset);
 757
 758	return val;
 759}
 760
 761/**
 762 * ipa_endpoint_init_hdr() - Initialize HDR endpoint configuration register
 763 * @endpoint:	Endpoint pointer
 764 *
 765 * We program QMAP endpoints so each packet received is preceded by a QMAP
 766 * header structure.  The QMAP header contains a 1-byte mux_id and 2-byte
 767 * packet size field, and we have the IPA hardware populate both for each
 768 * received packet.  The header is configured (in the HDR_EXT register)
 769 * to use big endian format.
 770 *
 771 * The packet size is written into the QMAP header's pkt_len field.  That
 772 * location is defined here using the HDR_OFST_PKT_SIZE field.
 773 *
 774 * The mux_id comes from a 4-byte metadata value supplied with each packet
 775 * by the modem.  It is *not* a QMAP header, but it does contain the mux_id
 776 * value that we want, in its low-order byte.  A bitmask defined in the
 777 * endpoint's METADATA_MASK register defines which byte within the modem
 778 * metadata contains the mux_id.  And the OFST_METADATA field programmed
 779 * here indicates where the extracted byte should be placed within the QMAP
 780 * header.
 781 */
 782static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint)
 783{
 784	u32 endpoint_id = endpoint->endpoint_id;
 785	struct ipa *ipa = endpoint->ipa;
 786	const struct reg *reg;
 787	u32 val = 0;
 788
 789	reg = ipa_reg(ipa, ENDP_INIT_HDR);
 790	if (endpoint->config.qmap) {
 791		enum ipa_version version = ipa->version;
 792		size_t header_size;
 793
 794		header_size = ipa_qmap_header_size(version, endpoint);
 795		val = ipa_header_size_encode(version, reg, header_size);
 796
 797		/* Define how to fill fields in a received QMAP header */
 798		if (!endpoint->toward_ipa) {
 799			u32 off;     /* Field offset within header */
 800
 801			/* Where IPA will write the metadata value */
 802			off = offsetof(struct rmnet_map_header, mux_id);
 803			val |= ipa_metadata_offset_encode(version, reg, off);
 804
 805			/* Where IPA will write the length */
 806			off = offsetof(struct rmnet_map_header, pkt_len);
 807			/* Upper bits are stored in HDR_EXT with IPA v4.5 */
 808			if (version >= IPA_VERSION_4_5)
 809				off &= reg_field_max(reg, HDR_OFST_PKT_SIZE);
 810
 811			val |= reg_bit(reg, HDR_OFST_PKT_SIZE_VALID);
 812			val |= reg_encode(reg, HDR_OFST_PKT_SIZE, off);
 813		}
 814		/* For QMAP TX, metadata offset is 0 (modem assumes this) */
 815		val |= reg_bit(reg, HDR_OFST_METADATA_VALID);
 816
 817		/* HDR_ADDITIONAL_CONST_LEN is 0; (RX only) */
 818		/* HDR_A5_MUX is 0 */
 819		/* HDR_LEN_INC_DEAGG_HDR is 0 */
 820		/* HDR_METADATA_REG_VALID is 0 (TX only, version < v4.5) */
 821	}
 822
 823	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
 824}
 825
 826static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint)
 827{
 828	u32 pad_align = endpoint->config.rx.pad_align;
 829	u32 endpoint_id = endpoint->endpoint_id;
 830	struct ipa *ipa = endpoint->ipa;
 831	const struct reg *reg;
 832	u32 val = 0;
 833
 834	reg = ipa_reg(ipa, ENDP_INIT_HDR_EXT);
 835	if (endpoint->config.qmap) {
 836		/* We have a header, so we must specify its endianness */
 837		val |= reg_bit(reg, HDR_ENDIANNESS);	/* big endian */
 838
 839		/* A QMAP header contains a 6 bit pad field at offset 0.
 840		 * The RMNet driver assumes this field is meaningful in
 841		 * packets it receives, and assumes the header's payload
 842		 * length includes that padding.  The RMNet driver does
 843		 * *not* pad packets it sends, however, so the pad field
 844		 * (although 0) should be ignored.
 845		 */
 846		if (!endpoint->toward_ipa) {
 847			val |= reg_bit(reg, HDR_TOTAL_LEN_OR_PAD_VALID);
 848			/* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */
 849			val |= reg_bit(reg, HDR_PAYLOAD_LEN_INC_PADDING);
 850			/* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */
 851		}
 852	}
 853
 854	/* HDR_PAYLOAD_LEN_INC_PADDING is 0 */
 855	if (!endpoint->toward_ipa)
 856		val |= reg_encode(reg, HDR_PAD_TO_ALIGNMENT, pad_align);
 857
 858	/* IPA v4.5 adds some most-significant bits to a few fields,
 859	 * two of which are defined in the HDR (not HDR_EXT) register.
 860	 */
 861	if (ipa->version >= IPA_VERSION_4_5) {
 862		/* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0, so MSB is 0 */
 863		if (endpoint->config.qmap && !endpoint->toward_ipa) {
 864			u32 mask = reg_field_max(reg, HDR_OFST_PKT_SIZE);
 865			u32 off;     /* Field offset within header */
 866
 867			off = offsetof(struct rmnet_map_header, pkt_len);
 868			/* Low bits are in the ENDP_INIT_HDR register */
 869			off >>= hweight32(mask);
 870			val |= reg_encode(reg, HDR_OFST_PKT_SIZE_MSB, off);
 871			/* HDR_ADDITIONAL_CONST_LEN is 0 so MSB is 0 */
 872		}
 873	}
 874
 875	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
 876}
 877
 878static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint)
 879{
 880	u32 endpoint_id = endpoint->endpoint_id;
 881	struct ipa *ipa = endpoint->ipa;
 882	const struct reg *reg;
 883	u32 val = 0;
 884	u32 offset;
 885
 886	if (endpoint->toward_ipa)
 887		return;		/* Register not valid for TX endpoints */
 888
 889	reg = ipa_reg(ipa,  ENDP_INIT_HDR_METADATA_MASK);
 890	offset = reg_n_offset(reg, endpoint_id);
 891
 892	/* Note that HDR_ENDIANNESS indicates big endian header fields */
 893	if (endpoint->config.qmap)
 894		val = (__force u32)cpu_to_be32(IPA_ENDPOINT_QMAP_METADATA_MASK);
 895
 896	iowrite32(val, ipa->reg_virt + offset);
 897}
 898
 899static void ipa_endpoint_init_mode(struct ipa_endpoint *endpoint)
 900{
 901	struct ipa *ipa = endpoint->ipa;
 902	const struct reg *reg;
 903	u32 offset;
 904	u32 val;
 905
 906	if (!endpoint->toward_ipa)
 907		return;		/* Register not valid for RX endpoints */
 908
 909	reg = ipa_reg(ipa, ENDP_INIT_MODE);
 910	if (endpoint->config.dma_mode) {
 911		enum ipa_endpoint_name name = endpoint->config.dma_endpoint;
 912		u32 dma_endpoint_id = ipa->name_map[name]->endpoint_id;
 913
 914		val = reg_encode(reg, ENDP_MODE, IPA_DMA);
 915		val |= reg_encode(reg, DEST_PIPE_INDEX, dma_endpoint_id);
 916	} else {
 917		val = reg_encode(reg, ENDP_MODE, IPA_BASIC);
 918	}
 919	/* All other bits unspecified (and 0) */
 920
 921	offset = reg_n_offset(reg, endpoint->endpoint_id);
 922	iowrite32(val, ipa->reg_virt + offset);
 923}
 924
 925/* For IPA v4.5+, times are expressed using Qtime.  A time is represented
 926 * at one of several available granularities, which are configured in
 927 * ipa_qtime_config().  Three (or, starting with IPA v5.0, four) pulse
 928 * generators are set up with different "tick" periods.  A Qtime value
 929 * encodes a tick count along with an indication of a pulse generator
 930 * (which has a fixed tick period).  Two pulse generators are always
 931 * available to the AP; a third is available starting with IPA v5.0.
 932 * This function determines which pulse generator most accurately
 933 * represents the time period provided, and returns the tick count to
 934 * use to represent that time.
 935 */
 936static u32
 937ipa_qtime_val(struct ipa *ipa, u32 microseconds, u32 max, u32 *select)
 938{
 939	u32 which = 0;
 940	u32 ticks;
 941
 942	/* Pulse generator 0 has 100 microsecond granularity */
 943	ticks = DIV_ROUND_CLOSEST(microseconds, 100);
 944	if (ticks <= max)
 945		goto out;
 946
 947	/* Pulse generator 1 has millisecond granularity */
 948	which = 1;
 949	ticks = DIV_ROUND_CLOSEST(microseconds, 1000);
 950	if (ticks <= max)
 951		goto out;
 952
 953	if (ipa->version >= IPA_VERSION_5_0) {
 954		/* Pulse generator 2 has 10 millisecond granularity */
 955		which = 2;
 956		ticks = DIV_ROUND_CLOSEST(microseconds, 100);
 957	}
 958	WARN_ON(ticks > max);
 959out:
 960	*select = which;
 961
 962	return ticks;
 963}
 964
 965/* Encode the aggregation timer limit (microseconds) based on IPA version */
 966static u32 aggr_time_limit_encode(struct ipa *ipa, const struct reg *reg,
 967				  u32 microseconds)
 968{
 969	u32 ticks;
 970	u32 max;
 971
 972	if (!microseconds)
 973		return 0;	/* Nothing to compute if time limit is 0 */
 974
 975	max = reg_field_max(reg, TIME_LIMIT);
 976	if (ipa->version >= IPA_VERSION_4_5) {
 977		u32 select;
 978
 979		ticks = ipa_qtime_val(ipa, microseconds, max, &select);
 980
 981		return reg_encode(reg, AGGR_GRAN_SEL, select) |
 982		       reg_encode(reg, TIME_LIMIT, ticks);
 983	}
 984
 985	/* We program aggregation granularity in ipa_hardware_config() */
 986	ticks = DIV_ROUND_CLOSEST(microseconds, IPA_AGGR_GRANULARITY);
 987	WARN(ticks > max, "aggr_time_limit too large (%u > %u usec)\n",
 988	     microseconds, max * IPA_AGGR_GRANULARITY);
 989
 990	return reg_encode(reg, TIME_LIMIT, ticks);
 991}
 992
 993static void ipa_endpoint_init_aggr(struct ipa_endpoint *endpoint)
 994{
 995	u32 endpoint_id = endpoint->endpoint_id;
 996	struct ipa *ipa = endpoint->ipa;
 997	const struct reg *reg;
 998	u32 val = 0;
 999
1000	reg = ipa_reg(ipa, ENDP_INIT_AGGR);
1001	if (endpoint->config.aggregation) {
1002		if (!endpoint->toward_ipa) {
1003			const struct ipa_endpoint_rx *rx_config;
1004			u32 buffer_size;
1005			u32 limit;
1006
1007			rx_config = &endpoint->config.rx;
1008			val |= reg_encode(reg, AGGR_EN, IPA_ENABLE_AGGR);
1009			val |= reg_encode(reg, AGGR_TYPE, IPA_GENERIC);
1010
1011			buffer_size = rx_config->buffer_size;
1012			limit = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD,
1013						 rx_config->aggr_hard_limit);
1014			val |= reg_encode(reg, BYTE_LIMIT, limit);
1015
1016			limit = rx_config->aggr_time_limit;
1017			val |= aggr_time_limit_encode(ipa, reg, limit);
1018
1019			/* AGGR_PKT_LIMIT is 0 (unlimited) */
1020
1021			if (rx_config->aggr_close_eof)
1022				val |= reg_bit(reg, SW_EOF_ACTIVE);
1023		} else {
1024			val |= reg_encode(reg, AGGR_EN, IPA_ENABLE_DEAGGR);
1025			val |= reg_encode(reg, AGGR_TYPE, IPA_QCMAP);
1026			/* other fields ignored */
1027		}
1028		/* AGGR_FORCE_CLOSE is 0 */
1029		/* AGGR_GRAN_SEL is 0 for IPA v4.5 */
1030	} else {
1031		val |= reg_encode(reg, AGGR_EN, IPA_BYPASS_AGGR);
1032		/* other fields ignored */
1033	}
1034
1035	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1036}
1037
1038/* The head-of-line blocking timer is defined as a tick count.  For
1039 * IPA version 4.5 the tick count is based on the Qtimer, which is
1040 * derived from the 19.2 MHz SoC XO clock.  For older IPA versions
1041 * each tick represents 128 cycles of the IPA core clock.
1042 *
1043 * Return the encoded value representing the timeout period provided
1044 * that should be written to the ENDP_INIT_HOL_BLOCK_TIMER register.
1045 */
1046static u32 hol_block_timer_encode(struct ipa *ipa, const struct reg *reg,
1047				  u32 microseconds)
1048{
1049	u32 width;
1050	u32 scale;
1051	u64 ticks;
1052	u64 rate;
1053	u32 high;
1054	u32 val;
1055
1056	if (!microseconds)
1057		return 0;	/* Nothing to compute if timer period is 0 */
1058
1059	if (ipa->version >= IPA_VERSION_4_5) {
1060		u32 max = reg_field_max(reg, TIMER_LIMIT);
1061		u32 select;
1062		u32 ticks;
1063
1064		ticks = ipa_qtime_val(ipa, microseconds, max, &select);
1065
1066		return reg_encode(reg, TIMER_GRAN_SEL, 1) |
1067		       reg_encode(reg, TIMER_LIMIT, ticks);
1068	}
1069
1070	/* Use 64 bit arithmetic to avoid overflow */
1071	rate = ipa_core_clock_rate(ipa);
1072	ticks = DIV_ROUND_CLOSEST(microseconds * rate, 128 * USEC_PER_SEC);
1073
1074	/* We still need the result to fit into the field */
1075	WARN_ON(ticks > reg_field_max(reg, TIMER_BASE_VALUE));
1076
1077	/* IPA v3.5.1 through v4.1 just record the tick count */
1078	if (ipa->version < IPA_VERSION_4_2)
1079		return reg_encode(reg, TIMER_BASE_VALUE, (u32)ticks);
1080
1081	/* For IPA v4.2, the tick count is represented by base and
1082	 * scale fields within the 32-bit timer register, where:
1083	 *     ticks = base << scale;
1084	 * The best precision is achieved when the base value is as
1085	 * large as possible.  Find the highest set bit in the tick
1086	 * count, and extract the number of bits in the base field
1087	 * such that high bit is included.
1088	 */
1089	high = fls(ticks);		/* 1..32 (or warning above) */
1090	width = hweight32(reg_fmask(reg, TIMER_BASE_VALUE));
1091	scale = high > width ? high - width : 0;
1092	if (scale) {
1093		/* If we're scaling, round up to get a closer result */
1094		ticks += 1 << (scale - 1);
1095		/* High bit was set, so rounding might have affected it */
1096		if (fls(ticks) != high)
1097			scale++;
1098	}
1099
1100	val = reg_encode(reg, TIMER_SCALE, scale);
1101	val |= reg_encode(reg, TIMER_BASE_VALUE, (u32)ticks >> scale);
1102
1103	return val;
1104}
1105
1106/* If microseconds is 0, timeout is immediate */
1107static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint,
1108					      u32 microseconds)
1109{
1110	u32 endpoint_id = endpoint->endpoint_id;
1111	struct ipa *ipa = endpoint->ipa;
1112	const struct reg *reg;
1113	u32 val;
1114
1115	/* This should only be changed when HOL_BLOCK_EN is disabled */
1116	reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_TIMER);
1117	val = hol_block_timer_encode(ipa, reg, microseconds);
1118
1119	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1120}
1121
1122static void
1123ipa_endpoint_init_hol_block_en(struct ipa_endpoint *endpoint, bool enable)
1124{
1125	u32 endpoint_id = endpoint->endpoint_id;
1126	struct ipa *ipa = endpoint->ipa;
1127	const struct reg *reg;
1128	u32 offset;
1129	u32 val;
1130
1131	reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_EN);
1132	offset = reg_n_offset(reg, endpoint_id);
1133	val = enable ? reg_bit(reg, HOL_BLOCK_EN) : 0;
1134
1135	iowrite32(val, ipa->reg_virt + offset);
1136
1137	/* When enabling, the register must be written twice for IPA v4.5+ */
1138	if (enable && ipa->version >= IPA_VERSION_4_5)
1139		iowrite32(val, ipa->reg_virt + offset);
1140}
1141
1142/* Assumes HOL_BLOCK is in disabled state */
1143static void ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint,
1144					       u32 microseconds)
1145{
1146	ipa_endpoint_init_hol_block_timer(endpoint, microseconds);
1147	ipa_endpoint_init_hol_block_en(endpoint, true);
1148}
1149
1150static void ipa_endpoint_init_hol_block_disable(struct ipa_endpoint *endpoint)
1151{
1152	ipa_endpoint_init_hol_block_en(endpoint, false);
1153}
1154
1155void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa)
1156{
1157	u32 endpoint_id = 0;
1158
1159	while (endpoint_id < ipa->endpoint_count) {
1160		struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++];
1161
1162		if (endpoint->toward_ipa || endpoint->ee_id != GSI_EE_MODEM)
1163			continue;
1164
1165		ipa_endpoint_init_hol_block_disable(endpoint);
1166		ipa_endpoint_init_hol_block_enable(endpoint, 0);
1167	}
1168}
1169
1170static void ipa_endpoint_init_deaggr(struct ipa_endpoint *endpoint)
1171{
1172	u32 endpoint_id = endpoint->endpoint_id;
1173	struct ipa *ipa = endpoint->ipa;
1174	const struct reg *reg;
1175	u32 val = 0;
1176
1177	if (!endpoint->toward_ipa)
1178		return;		/* Register not valid for RX endpoints */
1179
1180	reg = ipa_reg(ipa, ENDP_INIT_DEAGGR);
1181	/* DEAGGR_HDR_LEN is 0 */
1182	/* PACKET_OFFSET_VALID is 0 */
1183	/* PACKET_OFFSET_LOCATION is ignored (not valid) */
1184	/* MAX_PACKET_LEN is 0 (not enforced) */
1185
1186	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1187}
1188
1189static void ipa_endpoint_init_rsrc_grp(struct ipa_endpoint *endpoint)
1190{
1191	u32 resource_group = endpoint->config.resource_group;
1192	u32 endpoint_id = endpoint->endpoint_id;
1193	struct ipa *ipa = endpoint->ipa;
1194	const struct reg *reg;
1195	u32 val;
1196
1197	reg = ipa_reg(ipa, ENDP_INIT_RSRC_GRP);
1198	val = reg_encode(reg, ENDP_RSRC_GRP, resource_group);
1199
1200	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1201}
1202
1203static void ipa_endpoint_init_seq(struct ipa_endpoint *endpoint)
1204{
1205	u32 endpoint_id = endpoint->endpoint_id;
1206	struct ipa *ipa = endpoint->ipa;
1207	const struct reg *reg;
1208	u32 val;
1209
1210	if (!endpoint->toward_ipa)
1211		return;		/* Register not valid for RX endpoints */
1212
1213	reg = ipa_reg(ipa, ENDP_INIT_SEQ);
1214
1215	/* Low-order byte configures primary packet processing */
1216	val = reg_encode(reg, SEQ_TYPE, endpoint->config.tx.seq_type);
1217
1218	/* Second byte (if supported) configures replicated packet processing */
1219	if (ipa->version < IPA_VERSION_4_5)
1220		val |= reg_encode(reg, SEQ_REP_TYPE,
1221				  endpoint->config.tx.seq_rep_type);
1222
1223	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1224}
1225
1226/**
1227 * ipa_endpoint_skb_tx() - Transmit a socket buffer
1228 * @endpoint:	Endpoint pointer
1229 * @skb:	Socket buffer to send
1230 *
1231 * Returns:	0 if successful, or a negative error code
1232 */
1233int ipa_endpoint_skb_tx(struct ipa_endpoint *endpoint, struct sk_buff *skb)
1234{
1235	struct gsi_trans *trans;
1236	u32 nr_frags;
1237	int ret;
1238
1239	/* Make sure source endpoint's TLV FIFO has enough entries to
1240	 * hold the linear portion of the skb and all its fragments.
1241	 * If not, see if we can linearize it before giving up.
1242	 */
1243	nr_frags = skb_shinfo(skb)->nr_frags;
1244	if (nr_frags > endpoint->skb_frag_max) {
1245		if (skb_linearize(skb))
1246			return -E2BIG;
1247		nr_frags = 0;
1248	}
1249
1250	trans = ipa_endpoint_trans_alloc(endpoint, 1 + nr_frags);
1251	if (!trans)
1252		return -EBUSY;
1253
1254	ret = gsi_trans_skb_add(trans, skb);
1255	if (ret)
1256		goto err_trans_free;
1257	trans->data = skb;	/* transaction owns skb now */
1258
1259	gsi_trans_commit(trans, !netdev_xmit_more());
1260
1261	return 0;
1262
1263err_trans_free:
1264	gsi_trans_free(trans);
1265
1266	return -ENOMEM;
1267}
1268
1269static void ipa_endpoint_status(struct ipa_endpoint *endpoint)
1270{
1271	u32 endpoint_id = endpoint->endpoint_id;
1272	struct ipa *ipa = endpoint->ipa;
1273	const struct reg *reg;
1274	u32 val = 0;
1275
1276	reg = ipa_reg(ipa, ENDP_STATUS);
1277	if (endpoint->config.status_enable) {
1278		val |= reg_bit(reg, STATUS_EN);
1279		if (endpoint->toward_ipa) {
1280			enum ipa_endpoint_name name;
1281			u32 status_endpoint_id;
1282
1283			name = endpoint->config.tx.status_endpoint;
1284			status_endpoint_id = ipa->name_map[name]->endpoint_id;
1285
1286			val |= reg_encode(reg, STATUS_ENDP, status_endpoint_id);
1287		}
1288		/* STATUS_LOCATION is 0, meaning IPA packet status
1289		 * precedes the packet (not present for IPA v4.5+)
1290		 */
1291		/* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v4.0+) */
1292	}
1293
1294	iowrite32(val, ipa->reg_virt + reg_n_offset(reg, endpoint_id));
1295}
1296
1297static int ipa_endpoint_replenish_one(struct ipa_endpoint *endpoint,
1298				      struct gsi_trans *trans)
1299{
1300	struct page *page;
1301	u32 buffer_size;
1302	u32 offset;
1303	u32 len;
1304	int ret;
1305
1306	buffer_size = endpoint->config.rx.buffer_size;
1307	page = dev_alloc_pages(get_order(buffer_size));
1308	if (!page)
1309		return -ENOMEM;
1310
1311	/* Offset the buffer to make space for skb headroom */
1312	offset = NET_SKB_PAD;
1313	len = buffer_size - offset;
1314
1315	ret = gsi_trans_page_add(trans, page, len, offset);
1316	if (ret)
1317		put_page(page);
1318	else
1319		trans->data = page;	/* transaction owns page now */
1320
1321	return ret;
1322}
1323
1324/**
1325 * ipa_endpoint_replenish() - Replenish endpoint receive buffers
1326 * @endpoint:	Endpoint to be replenished
1327 *
1328 * The IPA hardware can hold a fixed number of receive buffers for an RX
1329 * endpoint, based on the number of entries in the underlying channel ring
1330 * buffer.  If an endpoint's "backlog" is non-zero, it indicates how many
1331 * more receive buffers can be supplied to the hardware.  Replenishing for
1332 * an endpoint can be disabled, in which case buffers are not queued to
1333 * the hardware.
1334 */
1335static void ipa_endpoint_replenish(struct ipa_endpoint *endpoint)
1336{
1337	struct gsi_trans *trans;
1338
1339	if (!test_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags))
1340		return;
1341
1342	/* Skip it if it's already active */
1343	if (test_and_set_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags))
1344		return;
1345
1346	while ((trans = ipa_endpoint_trans_alloc(endpoint, 1))) {
1347		bool doorbell;
1348
1349		if (ipa_endpoint_replenish_one(endpoint, trans))
1350			goto try_again_later;
1351
1352
1353		/* Ring the doorbell if we've got a full batch */
1354		doorbell = !(++endpoint->replenish_count % IPA_REPLENISH_BATCH);
1355		gsi_trans_commit(trans, doorbell);
1356	}
1357
1358	clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
1359
1360	return;
1361
1362try_again_later:
1363	gsi_trans_free(trans);
1364	clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
1365
1366	/* Whenever a receive buffer transaction completes we'll try to
1367	 * replenish again.  It's unlikely, but if we fail to supply even
1368	 * one buffer, nothing will trigger another replenish attempt.
1369	 * If the hardware has no receive buffers queued, schedule work to
1370	 * try replenishing again.
1371	 */
1372	if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id))
1373		schedule_delayed_work(&endpoint->replenish_work,
1374				      msecs_to_jiffies(1));
1375}
1376
1377static void ipa_endpoint_replenish_enable(struct ipa_endpoint *endpoint)
1378{
1379	set_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
1380
1381	/* Start replenishing if hardware currently has no buffers */
1382	if (gsi_channel_trans_idle(&endpoint->ipa->gsi, endpoint->channel_id))
1383		ipa_endpoint_replenish(endpoint);
1384}
1385
1386static void ipa_endpoint_replenish_disable(struct ipa_endpoint *endpoint)
1387{
1388	clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
1389}
1390
1391static void ipa_endpoint_replenish_work(struct work_struct *work)
1392{
1393	struct delayed_work *dwork = to_delayed_work(work);
1394	struct ipa_endpoint *endpoint;
1395
1396	endpoint = container_of(dwork, struct ipa_endpoint, replenish_work);
1397
1398	ipa_endpoint_replenish(endpoint);
1399}
1400
1401static void ipa_endpoint_skb_copy(struct ipa_endpoint *endpoint,
1402				  void *data, u32 len, u32 extra)
1403{
1404	struct sk_buff *skb;
1405
1406	if (!endpoint->netdev)
1407		return;
1408
1409	skb = __dev_alloc_skb(len, GFP_ATOMIC);
1410	if (skb) {
1411		/* Copy the data into the socket buffer and receive it */
1412		skb_put(skb, len);
1413		memcpy(skb->data, data, len);
1414		skb->truesize += extra;
1415	}
1416
1417	ipa_modem_skb_rx(endpoint->netdev, skb);
1418}
1419
1420static bool ipa_endpoint_skb_build(struct ipa_endpoint *endpoint,
1421				   struct page *page, u32 len)
1422{
1423	u32 buffer_size = endpoint->config.rx.buffer_size;
1424	struct sk_buff *skb;
1425
1426	/* Nothing to do if there's no netdev */
1427	if (!endpoint->netdev)
1428		return false;
1429
1430	WARN_ON(len > SKB_WITH_OVERHEAD(buffer_size - NET_SKB_PAD));
1431
1432	skb = build_skb(page_address(page), buffer_size);
1433	if (skb) {
1434		/* Reserve the headroom and account for the data */
1435		skb_reserve(skb, NET_SKB_PAD);
1436		skb_put(skb, len);
1437	}
1438
1439	/* Receive the buffer (or record drop if unable to build it) */
1440	ipa_modem_skb_rx(endpoint->netdev, skb);
1441
1442	return skb != NULL;
1443}
1444
1445 /* The format of an IPA packet status structure is the same for several
1446  * status types (opcodes).  Other types aren't currently supported.
1447 */
1448static bool ipa_status_format_packet(enum ipa_status_opcode opcode)
1449{
1450	switch (opcode) {
1451	case IPA_STATUS_OPCODE_PACKET:
1452	case IPA_STATUS_OPCODE_DROPPED_PACKET:
1453	case IPA_STATUS_OPCODE_SUSPENDED_PACKET:
1454	case IPA_STATUS_OPCODE_PACKET_2ND_PASS:
1455		return true;
1456	default:
1457		return false;
1458	}
1459}
1460
1461static bool
1462ipa_endpoint_status_skip(struct ipa_endpoint *endpoint, const void *data)
1463{
1464	struct ipa *ipa = endpoint->ipa;
1465	enum ipa_status_opcode opcode;
1466	u32 endpoint_id;
1467
1468	opcode = ipa_status_extract(ipa, data, STATUS_OPCODE);
1469	if (!ipa_status_format_packet(opcode))
1470		return true;
1471
1472	endpoint_id = ipa_status_extract(ipa, data, STATUS_DST_ENDPOINT);
1473	if (endpoint_id != endpoint->endpoint_id)
1474		return true;
1475
1476	return false;	/* Don't skip this packet, process it */
1477}
1478
1479static bool
1480ipa_endpoint_status_tag_valid(struct ipa_endpoint *endpoint, const void *data)
1481{
1482	struct ipa_endpoint *command_endpoint;
1483	enum ipa_status_mask status_mask;
1484	struct ipa *ipa = endpoint->ipa;
1485	u32 endpoint_id;
1486
1487	status_mask = ipa_status_extract(ipa, data, STATUS_MASK);
1488	if (!status_mask)
1489		return false;	/* No valid tag */
1490
1491	/* The status contains a valid tag.  We know the packet was sent to
1492	 * this endpoint (already verified by ipa_endpoint_status_skip()).
1493	 * If the packet came from the AP->command TX endpoint we know
1494	 * this packet was sent as part of the pipeline clear process.
1495	 */
1496	endpoint_id = ipa_status_extract(ipa, data, STATUS_SRC_ENDPOINT);
1497	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];
1498	if (endpoint_id == command_endpoint->endpoint_id) {
1499		complete(&ipa->completion);
1500	} else {
1501		dev_err(&ipa->pdev->dev,
1502			"unexpected tagged packet from endpoint %u\n",
1503			endpoint_id);
1504	}
1505
1506	return true;
1507}
1508
1509/* Return whether the status indicates the packet should be dropped */
1510static bool
1511ipa_endpoint_status_drop(struct ipa_endpoint *endpoint, const void *data)
1512{
1513	enum ipa_status_exception exception;
1514	struct ipa *ipa = endpoint->ipa;
1515	u32 rule;
1516
1517	/* If the status indicates a tagged transfer, we'll drop the packet */
1518	if (ipa_endpoint_status_tag_valid(endpoint, data))
1519		return true;
1520
1521	/* Deaggregation exceptions we drop; all other types we consume */
1522	exception = ipa_status_extract(ipa, data, STATUS_EXCEPTION);
1523	if (exception)
1524		return exception == IPA_STATUS_EXCEPTION_DEAGGR;
1525
1526	/* Drop the packet if it fails to match a routing rule; otherwise no */
1527	rule = ipa_status_extract(ipa, data, STATUS_ROUTER_RULE_INDEX);
1528
1529	return rule == IPA_STATUS_RULE_MISS;
1530}
1531
1532static void ipa_endpoint_status_parse(struct ipa_endpoint *endpoint,
1533				      struct page *page, u32 total_len)
1534{
1535	u32 buffer_size = endpoint->config.rx.buffer_size;
1536	void *data = page_address(page) + NET_SKB_PAD;
1537	u32 unused = buffer_size - total_len;
1538	struct ipa *ipa = endpoint->ipa;
 
1539	u32 resid = total_len;
1540
1541	while (resid) {
1542		u32 length;
1543		u32 align;
1544		u32 len;
1545
1546		if (resid < IPA_STATUS_SIZE) {
1547			dev_err(&endpoint->ipa->pdev->dev,
1548				"short message (%u bytes < %zu byte status)\n",
1549				resid, IPA_STATUS_SIZE);
1550			break;
1551		}
1552
1553		/* Skip over status packets that lack packet data */
1554		length = ipa_status_extract(ipa, data, STATUS_LENGTH);
1555		if (!length || ipa_endpoint_status_skip(endpoint, data)) {
1556			data += IPA_STATUS_SIZE;
1557			resid -= IPA_STATUS_SIZE;
1558			continue;
1559		}
1560
1561		/* Compute the amount of buffer space consumed by the packet,
1562		 * including the status.  If the hardware is configured to
1563		 * pad packet data to an aligned boundary, account for that.
1564		 * And if checksum offload is enabled a trailer containing
1565		 * computed checksum information will be appended.
1566		 */
1567		align = endpoint->config.rx.pad_align ? : 1;
1568		len = IPA_STATUS_SIZE + ALIGN(length, align);
1569		if (endpoint->config.checksum)
1570			len += sizeof(struct rmnet_map_dl_csum_trailer);
1571
1572		if (!ipa_endpoint_status_drop(endpoint, data)) {
1573			void *data2;
1574			u32 extra;
1575
1576			/* Client receives only packet data (no status) */
1577			data2 = data + IPA_STATUS_SIZE;
1578
1579			/* Have the true size reflect the extra unused space in
1580			 * the original receive buffer.  Distribute the "cost"
1581			 * proportionately across all aggregated packets in the
1582			 * buffer.
1583			 */
1584			extra = DIV_ROUND_CLOSEST(unused * len, total_len);
1585			ipa_endpoint_skb_copy(endpoint, data2, length, extra);
1586		}
1587
1588		/* Consume status and the full packet it describes */
1589		data += len;
1590		resid -= len;
1591	}
1592}
1593
1594void ipa_endpoint_trans_complete(struct ipa_endpoint *endpoint,
1595				 struct gsi_trans *trans)
1596{
1597	struct page *page;
1598
1599	if (endpoint->toward_ipa)
1600		return;
1601
1602	if (trans->cancelled)
1603		goto done;
1604
1605	/* Parse or build a socket buffer using the actual received length */
1606	page = trans->data;
1607	if (endpoint->config.status_enable)
1608		ipa_endpoint_status_parse(endpoint, page, trans->len);
1609	else if (ipa_endpoint_skb_build(endpoint, page, trans->len))
1610		trans->data = NULL;	/* Pages have been consumed */
1611done:
1612	ipa_endpoint_replenish(endpoint);
1613}
1614
1615void ipa_endpoint_trans_release(struct ipa_endpoint *endpoint,
1616				struct gsi_trans *trans)
1617{
1618	if (endpoint->toward_ipa) {
1619		struct ipa *ipa = endpoint->ipa;
1620
1621		/* Nothing to do for command transactions */
1622		if (endpoint != ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]) {
1623			struct sk_buff *skb = trans->data;
1624
1625			if (skb)
1626				dev_kfree_skb_any(skb);
1627		}
1628	} else {
1629		struct page *page = trans->data;
1630
1631		if (page)
1632			put_page(page);
1633	}
1634}
1635
1636void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id)
1637{
1638	const struct reg *reg;
1639	u32 val;
1640
1641	reg = ipa_reg(ipa, ROUTE);
1642	/* ROUTE_DIS is 0 */
1643	val = reg_encode(reg, ROUTE_DEF_PIPE, endpoint_id);
1644	val |= reg_bit(reg, ROUTE_DEF_HDR_TABLE);
1645	/* ROUTE_DEF_HDR_OFST is 0 */
1646	val |= reg_encode(reg, ROUTE_FRAG_DEF_PIPE, endpoint_id);
1647	val |= reg_bit(reg, ROUTE_DEF_RETAIN_HDR);
1648
1649	iowrite32(val, ipa->reg_virt + reg_offset(reg));
1650}
1651
1652void ipa_endpoint_default_route_clear(struct ipa *ipa)
1653{
1654	ipa_endpoint_default_route_set(ipa, 0);
1655}
1656
1657/**
1658 * ipa_endpoint_reset_rx_aggr() - Reset RX endpoint with aggregation active
1659 * @endpoint:	Endpoint to be reset
1660 *
1661 * If aggregation is active on an RX endpoint when a reset is performed
1662 * on its underlying GSI channel, a special sequence of actions must be
1663 * taken to ensure the IPA pipeline is properly cleared.
1664 *
1665 * Return:	0 if successful, or a negative error code
1666 */
1667static int ipa_endpoint_reset_rx_aggr(struct ipa_endpoint *endpoint)
1668{
1669	struct device *dev = &endpoint->ipa->pdev->dev;
1670	struct ipa *ipa = endpoint->ipa;
 
1671	struct gsi *gsi = &ipa->gsi;
1672	bool suspended = false;
1673	dma_addr_t addr;
1674	u32 retries;
1675	u32 len = 1;
1676	void *virt;
1677	int ret;
1678
1679	virt = kzalloc(len, GFP_KERNEL);
1680	if (!virt)
1681		return -ENOMEM;
1682
1683	addr = dma_map_single(dev, virt, len, DMA_FROM_DEVICE);
1684	if (dma_mapping_error(dev, addr)) {
1685		ret = -ENOMEM;
1686		goto out_kfree;
1687	}
1688
1689	/* Force close aggregation before issuing the reset */
1690	ipa_endpoint_force_close(endpoint);
1691
1692	/* Reset and reconfigure the channel with the doorbell engine
1693	 * disabled.  Then poll until we know aggregation is no longer
1694	 * active.  We'll re-enable the doorbell (if appropriate) when
1695	 * we reset again below.
1696	 */
1697	gsi_channel_reset(gsi, endpoint->channel_id, false);
1698
1699	/* Make sure the channel isn't suspended */
1700	suspended = ipa_endpoint_program_suspend(endpoint, false);
1701
1702	/* Start channel and do a 1 byte read */
1703	ret = gsi_channel_start(gsi, endpoint->channel_id);
1704	if (ret)
1705		goto out_suspend_again;
1706
1707	ret = gsi_trans_read_byte(gsi, endpoint->channel_id, addr);
1708	if (ret)
1709		goto err_endpoint_stop;
1710
1711	/* Wait for aggregation to be closed on the channel */
1712	retries = IPA_ENDPOINT_RESET_AGGR_RETRY_MAX;
1713	do {
1714		if (!ipa_endpoint_aggr_active(endpoint))
1715			break;
1716		usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
1717	} while (retries--);
1718
1719	/* Check one last time */
1720	if (ipa_endpoint_aggr_active(endpoint))
1721		dev_err(dev, "endpoint %u still active during reset\n",
1722			endpoint->endpoint_id);
1723
1724	gsi_trans_read_byte_done(gsi, endpoint->channel_id);
1725
1726	ret = gsi_channel_stop(gsi, endpoint->channel_id);
1727	if (ret)
1728		goto out_suspend_again;
1729
1730	/* Finally, reset and reconfigure the channel again (re-enabling
1731	 * the doorbell engine if appropriate).  Sleep for 1 millisecond to
1732	 * complete the channel reset sequence.  Finish by suspending the
1733	 * channel again (if necessary).
1734	 */
1735	gsi_channel_reset(gsi, endpoint->channel_id, true);
1736
1737	usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
1738
1739	goto out_suspend_again;
1740
1741err_endpoint_stop:
1742	(void)gsi_channel_stop(gsi, endpoint->channel_id);
1743out_suspend_again:
1744	if (suspended)
1745		(void)ipa_endpoint_program_suspend(endpoint, true);
1746	dma_unmap_single(dev, addr, len, DMA_FROM_DEVICE);
1747out_kfree:
1748	kfree(virt);
1749
1750	return ret;
1751}
1752
1753static void ipa_endpoint_reset(struct ipa_endpoint *endpoint)
1754{
1755	u32 channel_id = endpoint->channel_id;
1756	struct ipa *ipa = endpoint->ipa;
1757	bool special;
1758	int ret = 0;
1759
1760	/* On IPA v3.5.1, if an RX endpoint is reset while aggregation
1761	 * is active, we need to handle things specially to recover.
1762	 * All other cases just need to reset the underlying GSI channel.
1763	 */
1764	special = ipa->version < IPA_VERSION_4_0 && !endpoint->toward_ipa &&
1765			endpoint->config.aggregation;
1766	if (special && ipa_endpoint_aggr_active(endpoint))
1767		ret = ipa_endpoint_reset_rx_aggr(endpoint);
1768	else
1769		gsi_channel_reset(&ipa->gsi, channel_id, true);
1770
1771	if (ret)
1772		dev_err(&ipa->pdev->dev,
1773			"error %d resetting channel %u for endpoint %u\n",
1774			ret, endpoint->channel_id, endpoint->endpoint_id);
1775}
1776
1777static void ipa_endpoint_program(struct ipa_endpoint *endpoint)
1778{
1779	if (endpoint->toward_ipa) {
1780		/* Newer versions of IPA use GSI channel flow control
1781		 * instead of endpoint DELAY mode to prevent sending data.
1782		 * Flow control is disabled for newly-allocated channels,
1783		 * and we can assume flow control is not (ever) enabled
1784		 * for AP TX channels.
1785		 */
1786		if (endpoint->ipa->version < IPA_VERSION_4_2)
1787			ipa_endpoint_program_delay(endpoint, false);
1788	} else {
1789		/* Ensure suspend mode is off on all AP RX endpoints */
1790		(void)ipa_endpoint_program_suspend(endpoint, false);
1791	}
1792	ipa_endpoint_init_cfg(endpoint);
1793	ipa_endpoint_init_nat(endpoint);
1794	ipa_endpoint_init_hdr(endpoint);
1795	ipa_endpoint_init_hdr_ext(endpoint);
1796	ipa_endpoint_init_hdr_metadata_mask(endpoint);
1797	ipa_endpoint_init_mode(endpoint);
1798	ipa_endpoint_init_aggr(endpoint);
1799	if (!endpoint->toward_ipa) {
1800		if (endpoint->config.rx.holb_drop)
1801			ipa_endpoint_init_hol_block_enable(endpoint, 0);
1802		else
1803			ipa_endpoint_init_hol_block_disable(endpoint);
1804	}
1805	ipa_endpoint_init_deaggr(endpoint);
1806	ipa_endpoint_init_rsrc_grp(endpoint);
1807	ipa_endpoint_init_seq(endpoint);
1808	ipa_endpoint_status(endpoint);
1809}
1810
1811int ipa_endpoint_enable_one(struct ipa_endpoint *endpoint)
1812{
1813	u32 endpoint_id = endpoint->endpoint_id;
1814	struct ipa *ipa = endpoint->ipa;
1815	struct gsi *gsi = &ipa->gsi;
1816	int ret;
1817
1818	ret = gsi_channel_start(gsi, endpoint->channel_id);
1819	if (ret) {
1820		dev_err(&ipa->pdev->dev,
1821			"error %d starting %cX channel %u for endpoint %u\n",
1822			ret, endpoint->toward_ipa ? 'T' : 'R',
1823			endpoint->channel_id, endpoint_id);
1824		return ret;
1825	}
1826
1827	if (!endpoint->toward_ipa) {
1828		ipa_interrupt_suspend_enable(ipa->interrupt, endpoint_id);
1829		ipa_endpoint_replenish_enable(endpoint);
1830	}
1831
1832	__set_bit(endpoint_id, ipa->enabled);
1833
1834	return 0;
1835}
1836
1837void ipa_endpoint_disable_one(struct ipa_endpoint *endpoint)
1838{
1839	u32 endpoint_id = endpoint->endpoint_id;
1840	struct ipa *ipa = endpoint->ipa;
1841	struct gsi *gsi = &ipa->gsi;
1842	int ret;
1843
1844	if (!test_bit(endpoint_id, ipa->enabled))
1845		return;
1846
1847	__clear_bit(endpoint_id, endpoint->ipa->enabled);
1848
1849	if (!endpoint->toward_ipa) {
1850		ipa_endpoint_replenish_disable(endpoint);
1851		ipa_interrupt_suspend_disable(ipa->interrupt, endpoint_id);
1852	}
1853
1854	/* Note that if stop fails, the channel's state is not well-defined */
1855	ret = gsi_channel_stop(gsi, endpoint->channel_id);
1856	if (ret)
1857		dev_err(&ipa->pdev->dev,
1858			"error %d attempting to stop endpoint %u\n", ret,
1859			endpoint_id);
1860}
1861
1862void ipa_endpoint_suspend_one(struct ipa_endpoint *endpoint)
1863{
1864	struct device *dev = &endpoint->ipa->pdev->dev;
1865	struct gsi *gsi = &endpoint->ipa->gsi;
1866	int ret;
1867
1868	if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled))
1869		return;
1870
1871	if (!endpoint->toward_ipa) {
1872		ipa_endpoint_replenish_disable(endpoint);
1873		(void)ipa_endpoint_program_suspend(endpoint, true);
1874	}
1875
1876	ret = gsi_channel_suspend(gsi, endpoint->channel_id);
1877	if (ret)
1878		dev_err(dev, "error %d suspending channel %u\n", ret,
1879			endpoint->channel_id);
1880}
1881
1882void ipa_endpoint_resume_one(struct ipa_endpoint *endpoint)
1883{
1884	struct device *dev = &endpoint->ipa->pdev->dev;
1885	struct gsi *gsi = &endpoint->ipa->gsi;
1886	int ret;
1887
1888	if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled))
1889		return;
1890
1891	if (!endpoint->toward_ipa)
1892		(void)ipa_endpoint_program_suspend(endpoint, false);
1893
1894	ret = gsi_channel_resume(gsi, endpoint->channel_id);
1895	if (ret)
1896		dev_err(dev, "error %d resuming channel %u\n", ret,
1897			endpoint->channel_id);
1898	else if (!endpoint->toward_ipa)
1899		ipa_endpoint_replenish_enable(endpoint);
1900}
1901
1902void ipa_endpoint_suspend(struct ipa *ipa)
1903{
1904	if (!ipa->setup_complete)
1905		return;
1906
1907	if (ipa->modem_netdev)
1908		ipa_modem_suspend(ipa->modem_netdev);
1909
1910	ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);
1911	ipa_endpoint_suspend_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);
1912}
1913
1914void ipa_endpoint_resume(struct ipa *ipa)
1915{
1916	if (!ipa->setup_complete)
1917		return;
1918
1919	ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);
1920	ipa_endpoint_resume_one(ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);
1921
1922	if (ipa->modem_netdev)
1923		ipa_modem_resume(ipa->modem_netdev);
1924}
1925
1926static void ipa_endpoint_setup_one(struct ipa_endpoint *endpoint)
1927{
1928	struct gsi *gsi = &endpoint->ipa->gsi;
1929	u32 channel_id = endpoint->channel_id;
1930
1931	/* Only AP endpoints get set up */
1932	if (endpoint->ee_id != GSI_EE_AP)
1933		return;
1934
1935	endpoint->skb_frag_max = gsi->channel[channel_id].trans_tre_max - 1;
1936	if (!endpoint->toward_ipa) {
1937		/* RX transactions require a single TRE, so the maximum
1938		 * backlog is the same as the maximum outstanding TREs.
1939		 */
1940		clear_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags);
1941		clear_bit(IPA_REPLENISH_ACTIVE, endpoint->replenish_flags);
1942		INIT_DELAYED_WORK(&endpoint->replenish_work,
1943				  ipa_endpoint_replenish_work);
1944	}
1945
1946	ipa_endpoint_program(endpoint);
1947
1948	__set_bit(endpoint->endpoint_id, endpoint->ipa->set_up);
1949}
1950
1951static void ipa_endpoint_teardown_one(struct ipa_endpoint *endpoint)
1952{
1953	__clear_bit(endpoint->endpoint_id, endpoint->ipa->set_up);
1954
1955	if (!endpoint->toward_ipa)
1956		cancel_delayed_work_sync(&endpoint->replenish_work);
1957
1958	ipa_endpoint_reset(endpoint);
1959}
1960
1961void ipa_endpoint_setup(struct ipa *ipa)
1962{
1963	u32 endpoint_id;
1964
1965	for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count)
1966		ipa_endpoint_setup_one(&ipa->endpoint[endpoint_id]);
1967}
1968
1969void ipa_endpoint_teardown(struct ipa *ipa)
1970{
1971	u32 endpoint_id;
1972
1973	for_each_set_bit(endpoint_id, ipa->set_up, ipa->endpoint_count)
1974		ipa_endpoint_teardown_one(&ipa->endpoint[endpoint_id]);
1975}
1976
1977void ipa_endpoint_deconfig(struct ipa *ipa)
1978{
1979	ipa->available_count = 0;
1980	bitmap_free(ipa->available);
1981	ipa->available = NULL;
1982}
1983
1984int ipa_endpoint_config(struct ipa *ipa)
1985{
1986	struct device *dev = &ipa->pdev->dev;
1987	const struct reg *reg;
1988	u32 endpoint_id;
1989	u32 hw_limit;
1990	u32 tx_count;
1991	u32 rx_count;
1992	u32 rx_base;
1993	u32 limit;
1994	u32 val;
1995
1996	/* Prior to IPA v3.5, the FLAVOR_0 register was not supported.
1997	 * Furthermore, the endpoints were not grouped such that TX
1998	 * endpoint numbers started with 0 and RX endpoints had numbers
1999	 * higher than all TX endpoints, so we can't do the simple
2000	 * direction check used for newer hardware below.
2001	 *
2002	 * For hardware that doesn't support the FLAVOR_0 register,
2003	 * just set the available mask to support any endpoint, and
2004	 * assume the configuration is valid.
2005	 */
2006	if (ipa->version < IPA_VERSION_3_5) {
2007		ipa->available = bitmap_zalloc(IPA_ENDPOINT_MAX, GFP_KERNEL);
2008		if (!ipa->available)
2009			return -ENOMEM;
2010		ipa->available_count = IPA_ENDPOINT_MAX;
2011
2012		bitmap_set(ipa->available, 0, IPA_ENDPOINT_MAX);
2013
2014		return 0;
2015	}
2016
2017	/* Find out about the endpoints supplied by the hardware, and ensure
2018	 * the highest one doesn't exceed the number supported by software.
2019	 */
2020	reg = ipa_reg(ipa, FLAVOR_0);
2021	val = ioread32(ipa->reg_virt + reg_offset(reg));
2022
2023	/* Our RX is an IPA producer; our TX is an IPA consumer. */
2024	tx_count = reg_decode(reg, MAX_CONS_PIPES, val);
2025	rx_count = reg_decode(reg, MAX_PROD_PIPES, val);
2026	rx_base = reg_decode(reg, PROD_LOWEST, val);
2027
2028	limit = rx_base + rx_count;
2029	if (limit > IPA_ENDPOINT_MAX) {
2030		dev_err(dev, "too many endpoints, %u > %u\n",
2031			limit, IPA_ENDPOINT_MAX);
2032		return -EINVAL;
2033	}
2034
2035	/* Until IPA v5.0, the max endpoint ID was 32 */
2036	hw_limit = ipa->version < IPA_VERSION_5_0 ? 32 : U8_MAX + 1;
2037	if (limit > hw_limit) {
2038		dev_err(dev, "unexpected endpoint count, %u > %u\n",
2039			limit, hw_limit);
2040		return -EINVAL;
2041	}
2042
2043	/* Allocate and initialize the available endpoint bitmap */
2044	ipa->available = bitmap_zalloc(limit, GFP_KERNEL);
2045	if (!ipa->available)
2046		return -ENOMEM;
2047	ipa->available_count = limit;
2048
2049	/* Mark all supported RX and TX endpoints as available */
2050	bitmap_set(ipa->available, 0, tx_count);
2051	bitmap_set(ipa->available, rx_base, rx_count);
2052
2053	for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) {
2054		struct ipa_endpoint *endpoint;
2055
2056		if (endpoint_id >= limit) {
2057			dev_err(dev, "invalid endpoint id, %u > %u\n",
2058				endpoint_id, limit - 1);
2059			goto err_free_bitmap;
2060		}
2061
2062		if (!test_bit(endpoint_id, ipa->available)) {
2063			dev_err(dev, "unavailable endpoint id %u\n",
2064				endpoint_id);
2065			goto err_free_bitmap;
2066		}
2067
2068		/* Make sure it's pointing in the right direction */
2069		endpoint = &ipa->endpoint[endpoint_id];
2070		if (endpoint->toward_ipa) {
2071			if (endpoint_id < tx_count)
2072				continue;
2073		} else if (endpoint_id >= rx_base) {
2074			continue;
2075		}
2076
2077		dev_err(dev, "endpoint id %u wrong direction\n", endpoint_id);
2078		goto err_free_bitmap;
2079	}
2080
2081	return 0;
2082
2083err_free_bitmap:
2084	ipa_endpoint_deconfig(ipa);
2085
2086	return -EINVAL;
2087}
2088
2089static void ipa_endpoint_init_one(struct ipa *ipa, enum ipa_endpoint_name name,
2090				  const struct ipa_gsi_endpoint_data *data)
2091{
2092	struct ipa_endpoint *endpoint;
2093
2094	endpoint = &ipa->endpoint[data->endpoint_id];
2095
2096	if (data->ee_id == GSI_EE_AP)
2097		ipa->channel_map[data->channel_id] = endpoint;
2098	ipa->name_map[name] = endpoint;
2099
2100	endpoint->ipa = ipa;
2101	endpoint->ee_id = data->ee_id;
2102	endpoint->channel_id = data->channel_id;
2103	endpoint->endpoint_id = data->endpoint_id;
2104	endpoint->toward_ipa = data->toward_ipa;
2105	endpoint->config = data->endpoint.config;
2106
2107	__set_bit(endpoint->endpoint_id, ipa->defined);
2108}
2109
2110static void ipa_endpoint_exit_one(struct ipa_endpoint *endpoint)
2111{
2112	__clear_bit(endpoint->endpoint_id, endpoint->ipa->defined);
2113
2114	memset(endpoint, 0, sizeof(*endpoint));
2115}
2116
2117void ipa_endpoint_exit(struct ipa *ipa)
2118{
2119	u32 endpoint_id;
2120
2121	ipa->filtered = 0;
2122
2123	for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count)
2124		ipa_endpoint_exit_one(&ipa->endpoint[endpoint_id]);
2125
2126	bitmap_free(ipa->enabled);
2127	ipa->enabled = NULL;
2128	bitmap_free(ipa->set_up);
2129	ipa->set_up = NULL;
2130	bitmap_free(ipa->defined);
2131	ipa->defined = NULL;
2132
2133	memset(ipa->name_map, 0, sizeof(ipa->name_map));
2134	memset(ipa->channel_map, 0, sizeof(ipa->channel_map));
2135}
2136
2137/* Returns a bitmask of endpoints that support filtering, or 0 on error */
2138int ipa_endpoint_init(struct ipa *ipa, u32 count,
2139		      const struct ipa_gsi_endpoint_data *data)
2140{
2141	enum ipa_endpoint_name name;
2142	u32 filtered;
2143
2144	BUILD_BUG_ON(!IPA_REPLENISH_BATCH);
2145
2146	/* Number of endpoints is one more than the maximum ID */
2147	ipa->endpoint_count = ipa_endpoint_max(ipa, count, data) + 1;
2148	if (!ipa->endpoint_count)
2149		return -EINVAL;
2150
2151	/* Initialize endpoint state bitmaps */
2152	ipa->defined = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL);
2153	if (!ipa->defined)
2154		return -ENOMEM;
2155
2156	ipa->set_up = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL);
2157	if (!ipa->set_up)
2158		goto err_free_defined;
2159
2160	ipa->enabled = bitmap_zalloc(ipa->endpoint_count, GFP_KERNEL);
2161	if (!ipa->enabled)
2162		goto err_free_set_up;
2163
2164	filtered = 0;
2165	for (name = 0; name < count; name++, data++) {
2166		if (ipa_gsi_endpoint_data_empty(data))
2167			continue;	/* Skip over empty slots */
2168
2169		ipa_endpoint_init_one(ipa, name, data);
2170
2171		if (data->endpoint.filter_support)
2172			filtered |= BIT(data->endpoint_id);
2173		if (data->ee_id == GSI_EE_MODEM && data->toward_ipa)
2174			ipa->modem_tx_count++;
2175	}
2176
2177	/* Make sure the set of filtered endpoints is valid */
2178	if (!ipa_filtered_valid(ipa, filtered)) {
2179		ipa_endpoint_exit(ipa);
2180
2181		return -EINVAL;
2182	}
2183
2184	ipa->filtered = filtered;
2185
2186	return 0;
2187
2188err_free_set_up:
2189	bitmap_free(ipa->set_up);
2190	ipa->set_up = NULL;
2191err_free_defined:
2192	bitmap_free(ipa->defined);
2193	ipa->defined = NULL;
2194
2195	return -ENOMEM;
2196}