1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * ARM Message Handling Unit Version 2 (MHUv2) driver.
   4 *
   5 * Copyright (C) 2020 ARM Ltd.
   6 * Copyright (C) 2020 Linaro Ltd.
   7 *
   8 * An MHUv2 mailbox controller can provide up to 124 channel windows (each 32
   9 * bit long) and the driver allows any combination of both the transport
  10 * protocol modes: data-transfer and doorbell, to be used on those channel
  11 * windows.
  12 *
  13 * The transport protocols should be specified in the device tree entry for the
  14 * device. The transport protocols determine how the underlying hardware
  15 * resources of the device are utilized when transmitting data. Refer to the
  16 * device tree bindings of the ARM MHUv2 controller for more details.
  17 *
  18 * The number of registered mailbox channels is dependent on both the underlying
  19 * hardware - mainly the number of channel windows implemented by the platform,
  20 * as well as the selected transport protocols.
  21 *
  22 * The MHUv2 controller can work both as a sender and receiver, but the driver
  23 * and the DT bindings support unidirectional transfers for better allocation of
  24 * the channels. That is, this driver will be probed for two separate devices
  25 * for each mailbox controller, a sender device and a receiver device.
  26 */
  27
  28#include <linux/amba/bus.h>
  29#include <linux/interrupt.h>
  30#include <linux/mailbox_controller.h>
  31#include <linux/mailbox/arm_mhuv2_message.h>
  32#include <linux/module.h>
  33#include <linux/of_address.h>
  34#include <linux/spinlock.h>
  35
  36/* ====== MHUv2 Registers ====== */
  37
  38/* Maximum number of channel windows */
  39#define MHUV2_CH_WN_MAX			124
  40/* Number of combined interrupt status registers */
  41#define MHUV2_CMB_INT_ST_REG_CNT	4
  42#define MHUV2_STAT_BYTES		(sizeof(u32))
  43#define MHUV2_STAT_BITS			(MHUV2_STAT_BYTES * __CHAR_BIT__)
  44
  45#define LSB_MASK(n)			((1 << (n * __CHAR_BIT__)) - 1)
  46#define MHUV2_PROTOCOL_PROP		"arm,mhuv2-protocols"
  47
  48/* Register Message Handling Unit Configuration fields */
  49struct mhu_cfg_t {
  50	u32 num_ch : 7;
  51	u32 pad : 25;
  52} __packed;
  53
  54/* register Interrupt Status fields */
  55struct int_st_t {
  56	u32 nr2r : 1;
  57	u32 r2nr : 1;
  58	u32 pad : 30;
  59} __packed;
  60
  61/* Register Interrupt Clear fields */
  62struct int_clr_t {
  63	u32 nr2r : 1;
  64	u32 r2nr : 1;
  65	u32 pad : 30;
  66} __packed;
  67
  68/* Register Interrupt Enable fields */
  69struct int_en_t {
  70	u32 r2nr : 1;
  71	u32 nr2r : 1;
  72	u32 chcomb : 1;
  73	u32 pad : 29;
  74} __packed;
  75
  76/* Register Implementer Identification fields */
  77struct iidr_t {
  78	u32 implementer : 12;
  79	u32 revision : 4;
  80	u32 variant : 4;
  81	u32 product_id : 12;
  82} __packed;
  83
  84/* Register Architecture Identification Register fields */
  85struct aidr_t {
  86	u32 arch_minor_rev : 4;
  87	u32 arch_major_rev : 4;
  88	u32 pad : 24;
  89} __packed;
  90
  91/* Sender Channel Window fields */
  92struct mhu2_send_ch_wn_reg {
  93	u32 stat;
  94	u8 pad1[0x0C - 0x04];
  95	u32 stat_set;
  96	u32 int_st;
  97	u32 int_clr;
  98	u32 int_en;
  99	u8 pad2[0x20 - 0x1C];
 100} __packed;
 101
 102/* Sender frame register fields */
 103struct mhu2_send_frame_reg {
 104	struct mhu2_send_ch_wn_reg ch_wn[MHUV2_CH_WN_MAX];
 105	struct mhu_cfg_t mhu_cfg;
 106	u32 resp_cfg;
 107	u32 access_request;
 108	u32 access_ready;
 109	struct int_st_t int_st;
 110	struct int_clr_t int_clr;
 111	struct int_en_t int_en;
 112	u32 reserved0;
 113	u32 chcomb_int_st[MHUV2_CMB_INT_ST_REG_CNT];
 114	u8 pad[0xFC8 - 0xFB0];
 115	struct iidr_t iidr;
 116	struct aidr_t aidr;
 117} __packed;
 118
 119/* Receiver Channel Window fields */
 120struct mhu2_recv_ch_wn_reg {
 121	u32 stat;
 122	u32 stat_masked;
 123	u32 stat_clear;
 124	u8 reserved0[0x10 - 0x0C];
 125	u32 mask;
 126	u32 mask_set;
 127	u32 mask_clear;
 128	u8 pad[0x20 - 0x1C];
 129} __packed;
 130
 131/* Receiver frame register fields */
 132struct mhu2_recv_frame_reg {
 133	struct mhu2_recv_ch_wn_reg ch_wn[MHUV2_CH_WN_MAX];
 134	struct mhu_cfg_t mhu_cfg;
 135	u8 reserved0[0xF90 - 0xF84];
 136	struct int_st_t int_st;
 137	struct int_clr_t int_clr;
 138	struct int_en_t int_en;
 139	u32 pad;
 140	u32 chcomb_int_st[MHUV2_CMB_INT_ST_REG_CNT];
 141	u8 reserved2[0xFC8 - 0xFB0];
 142	struct iidr_t iidr;
 143	struct aidr_t aidr;
 144} __packed;
 145
 146
 147/* ====== MHUv2 data structures ====== */
 148
 149enum mhuv2_transport_protocol {
 150	DOORBELL = 0,
 151	DATA_TRANSFER = 1
 152};
 153
 154enum mhuv2_frame {
 155	RECEIVER_FRAME,
 156	SENDER_FRAME
 157};
 158
 159/**
 160 * struct mhuv2 - MHUv2 mailbox controller data
 161 *
 162 * @mbox:	Mailbox controller belonging to the MHU frame.
 163 * @send:	Base address of the register mapping region.
 164 * @recv:	Base address of the register mapping region.
 165 * @frame:	Frame type: RECEIVER_FRAME or SENDER_FRAME.
 166 * @irq:	Interrupt.
 167 * @windows:	Channel windows implemented by the platform.
 168 * @minor:	Minor version of the controller.
 169 * @length:	Length of the protocols array in bytes.
 170 * @protocols:	Raw protocol information, derived from device tree.
 171 * @doorbell_pending_lock: spinlock required for correct operation of Tx
 172 *		interrupt for doorbells.
 173 */
 174struct mhuv2 {
 175	struct mbox_controller mbox;
 176	union {
 177		struct mhu2_send_frame_reg __iomem *send;
 178		struct mhu2_recv_frame_reg __iomem *recv;
 179	};
 180	enum mhuv2_frame frame;
 181	unsigned int irq;
 182	unsigned int windows;
 183	unsigned int minor;
 184	unsigned int length;
 185	u32 *protocols;
 186
 187	spinlock_t doorbell_pending_lock;
 188};
 189
 190#define mhu_from_mbox(_mbox) container_of(_mbox, struct mhuv2, mbox)
 191
 192/**
 193 * struct mhuv2_protocol_ops - MHUv2 operations
 194 *
 195 * Each transport protocol must provide an implementation of the operations
 196 * provided here.
 197 *
 198 * @rx_startup: Startup callback for receiver.
 199 * @rx_shutdown: Shutdown callback for receiver.
 200 * @read_data: Reads and clears newly available data.
 201 * @tx_startup: Startup callback for receiver.
 202 * @tx_shutdown: Shutdown callback for receiver.
 203 * @last_tx_done: Report back if the last tx is completed or not.
 204 * @send_data: Send data to the receiver.
 205 */
 206struct mhuv2_protocol_ops {
 207	int (*rx_startup)(struct mhuv2 *mhu, struct mbox_chan *chan);
 208	void (*rx_shutdown)(struct mhuv2 *mhu, struct mbox_chan *chan);
 209	void *(*read_data)(struct mhuv2 *mhu, struct mbox_chan *chan);
 210
 211	void (*tx_startup)(struct mhuv2 *mhu, struct mbox_chan *chan);
 212	void (*tx_shutdown)(struct mhuv2 *mhu, struct mbox_chan *chan);
 213	int (*last_tx_done)(struct mhuv2 *mhu, struct mbox_chan *chan);
 214	int (*send_data)(struct mhuv2 *mhu, struct mbox_chan *chan, void *arg);
 215};
 216
 217/*
 218 * MHUv2 mailbox channel's private information
 219 *
 220 * @ops:	protocol specific ops for the channel.
 221 * @ch_wn_idx:	Channel window index allocated to the channel.
 222 * @windows:	Total number of windows consumed by the channel, only relevant
 223 *		in DATA_TRANSFER protocol.
 224 * @doorbell:	Doorbell bit number within the ch_wn_idx window, only relevant
 225 *		in DOORBELL protocol.
 226 * @pending:	Flag indicating pending doorbell interrupt, only relevant in
 227 *		DOORBELL protocol.
 228 */
 229struct mhuv2_mbox_chan_priv {
 230	const struct mhuv2_protocol_ops *ops;
 231	u32 ch_wn_idx;
 232	union {
 233		u32 windows;
 234		struct {
 235			u32 doorbell;
 236			u32 pending;
 237		};
 238	};
 239};
 240
 241/* Macro for reading a bitfield within a physically mapped packed struct */
 242#define readl_relaxed_bitfield(_regptr, _type, _field)			\
 243	({								\
 244		u32 _regval;						\
 245		_regval = readl_relaxed((_regptr));			\
 246		(*(_type *)(&_regval))._field;				\
 247	})
 248
 249/* Macro for writing a bitfield within a physically mapped packed struct */
 250#define writel_relaxed_bitfield(_value, _regptr, _type, _field)		\
 251	({								\
 252		u32 _regval;						\
 253		_regval = readl_relaxed(_regptr);			\
 254		(*(_type *)(&_regval))._field = _value;			\
 255		writel_relaxed(_regval, _regptr);			\
 256	})
 257
 258
 259/* =================== Doorbell transport protocol operations =============== */
 260
 261static int mhuv2_doorbell_rx_startup(struct mhuv2 *mhu, struct mbox_chan *chan)
 262{
 263	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 264
 265	writel_relaxed(BIT(priv->doorbell),
 266		       &mhu->recv->ch_wn[priv->ch_wn_idx].mask_clear);
 267	return 0;
 268}
 269
 270static void mhuv2_doorbell_rx_shutdown(struct mhuv2 *mhu,
 271				       struct mbox_chan *chan)
 272{
 273	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 274
 275	writel_relaxed(BIT(priv->doorbell),
 276		       &mhu->recv->ch_wn[priv->ch_wn_idx].mask_set);
 277}
 278
 279static void *mhuv2_doorbell_read_data(struct mhuv2 *mhu, struct mbox_chan *chan)
 280{
 281	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 282
 283	writel_relaxed(BIT(priv->doorbell),
 284		       &mhu->recv->ch_wn[priv->ch_wn_idx].stat_clear);
 285	return NULL;
 286}
 287
 288static int mhuv2_doorbell_last_tx_done(struct mhuv2 *mhu,
 289				       struct mbox_chan *chan)
 290{
 291	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 292
 293	return !(readl_relaxed(&mhu->send->ch_wn[priv->ch_wn_idx].stat) &
 294		 BIT(priv->doorbell));
 295}
 296
 297static int mhuv2_doorbell_send_data(struct mhuv2 *mhu, struct mbox_chan *chan,
 298				    void *arg)
 299{
 300	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 301	unsigned long flags;
 302
 303	spin_lock_irqsave(&mhu->doorbell_pending_lock, flags);
 304
 305	priv->pending = 1;
 306	writel_relaxed(BIT(priv->doorbell),
 307		       &mhu->send->ch_wn[priv->ch_wn_idx].stat_set);
 308
 309	spin_unlock_irqrestore(&mhu->doorbell_pending_lock, flags);
 310
 311	return 0;
 312}
 313
 314static const struct mhuv2_protocol_ops mhuv2_doorbell_ops = {
 315	.rx_startup = mhuv2_doorbell_rx_startup,
 316	.rx_shutdown = mhuv2_doorbell_rx_shutdown,
 317	.read_data = mhuv2_doorbell_read_data,
 318	.last_tx_done = mhuv2_doorbell_last_tx_done,
 319	.send_data = mhuv2_doorbell_send_data,
 320};
 321#define IS_PROTOCOL_DOORBELL(_priv) (_priv->ops == &mhuv2_doorbell_ops)
 322
 323/* ============= Data transfer transport protocol operations ================ */
 324
 325static int mhuv2_data_transfer_rx_startup(struct mhuv2 *mhu,
 326					  struct mbox_chan *chan)
 327{
 328	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 329	int i = priv->ch_wn_idx + priv->windows - 1;
 330
 331	/*
 332	 * The protocol mandates that all but the last status register must be
 333	 * masked.
 334	 */
 335	writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_clear);
 336	return 0;
 337}
 338
 339static void mhuv2_data_transfer_rx_shutdown(struct mhuv2 *mhu,
 340					    struct mbox_chan *chan)
 341{
 342	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 343	int i = priv->ch_wn_idx + priv->windows - 1;
 344
 345	writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_set);
 346}
 347
 348static void *mhuv2_data_transfer_read_data(struct mhuv2 *mhu,
 349					   struct mbox_chan *chan)
 350{
 351	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 352	const int windows = priv->windows;
 353	struct arm_mhuv2_mbox_msg *msg;
 354	u32 *data;
 355	int i, idx;
 356
 357	msg = kzalloc(sizeof(*msg) + windows * MHUV2_STAT_BYTES, GFP_KERNEL);
 358	if (!msg)
 359		return ERR_PTR(-ENOMEM);
 360
 361	data = msg->data = msg + 1;
 362	msg->len = windows * MHUV2_STAT_BYTES;
 363
 364	/*
 365	 * Messages are expected in order of most significant word to least
 366	 * significant word. Refer mhuv2_data_transfer_send_data() for more
 367	 * details.
 368	 *
 369	 * We also need to read the stat register instead of stat_masked, as we
 370	 * masked all but the last window.
 371	 *
 372	 * Last channel window must be cleared as the final operation. Upon
 373	 * clearing the last channel window register, which is unmasked in
 374	 * data-transfer protocol, the interrupt is de-asserted.
 375	 */
 376	for (i = 0; i < windows; i++) {
 377		idx = priv->ch_wn_idx + i;
 378		data[windows - 1 - i] = readl_relaxed(&mhu->recv->ch_wn[idx].stat);
 379		writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[idx].stat_clear);
 380	}
 381
 382	return msg;
 383}
 384
 385static void mhuv2_data_transfer_tx_startup(struct mhuv2 *mhu,
 386					   struct mbox_chan *chan)
 387{
 388	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 389	int i = priv->ch_wn_idx + priv->windows - 1;
 390
 391	/* Enable interrupts only for the last window */
 392	if (mhu->minor) {
 393		writel_relaxed(0x1, &mhu->send->ch_wn[i].int_clr);
 394		writel_relaxed(0x1, &mhu->send->ch_wn[i].int_en);
 395	}
 396}
 397
 398static void mhuv2_data_transfer_tx_shutdown(struct mhuv2 *mhu,
 399					    struct mbox_chan *chan)
 400{
 401	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 402	int i = priv->ch_wn_idx + priv->windows - 1;
 403
 404	if (mhu->minor)
 405		writel_relaxed(0x0, &mhu->send->ch_wn[i].int_en);
 406}
 407
 408static int mhuv2_data_transfer_last_tx_done(struct mhuv2 *mhu,
 409					    struct mbox_chan *chan)
 410{
 411	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 412	int i = priv->ch_wn_idx + priv->windows - 1;
 413
 414	/* Just checking the last channel window should be enough */
 415	return !readl_relaxed(&mhu->send->ch_wn[i].stat);
 416}
 417
 418/*
 419 * Message will be transmitted from most significant to least significant word.
 420 * This is to allow for messages shorter than channel windows to still trigger
 421 * the receiver interrupt which gets activated when the last stat register is
 422 * written. As an example, a 6-word message is to be written on a 4-channel MHU
 423 * connection: Registers marked with '*' are masked, and will not generate an
 424 * interrupt on the receiver side once written.
 425 *
 426 * u32 *data =	[0x00000001], [0x00000002], [0x00000003], [0x00000004],
 427 *		[0x00000005], [0x00000006]
 428 *
 429 * ROUND 1:
 430 * stat reg		To write	Write sequence
 431 * [ stat 3 ]	<-	[0x00000001]	4 <- triggers interrupt on receiver
 432 * [ stat 2 ]	<-	[0x00000002]	3
 433 * [ stat 1 ]	<-	[0x00000003]	2
 434 * [ stat 0 ]	<-	[0x00000004]	1
 435 *
 436 * data += 4 // Increment data pointer by number of stat regs
 437 *
 438 * ROUND 2:
 439 * stat reg		To write	Write sequence
 440 * [ stat 3 ]	<-	[0x00000005]	2 <- triggers interrupt on receiver
 441 * [ stat 2 ]	<-	[0x00000006]	1
 442 * [ stat 1 ]	<-	[0x00000000]
 443 * [ stat 0 ]	<-	[0x00000000]
 444 */
 445static int mhuv2_data_transfer_send_data(struct mhuv2 *mhu,
 446					 struct mbox_chan *chan, void *arg)
 447{
 448	const struct arm_mhuv2_mbox_msg *msg = arg;
 449	int bytes_left = msg->len, bytes_to_send, bytes_in_round, i;
 450	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 451	int windows = priv->windows;
 452	u32 *data = msg->data, word;
 453
 454	while (bytes_left) {
 455		if (!data[0]) {
 456			dev_err(mhu->mbox.dev, "Data aligned at first window can't be zero to guarantee interrupt generation at receiver");
 457			return -EINVAL;
 458		}
 459
 460		while(!mhuv2_data_transfer_last_tx_done(mhu, chan))
 461			continue;
 462
 463		bytes_in_round = min(bytes_left, (int)(windows * MHUV2_STAT_BYTES));
 464
 465		for (i = windows - 1; i >= 0; i--) {
 466			/* Data less than windows can transfer ? */
 467			if (unlikely(bytes_in_round <= i * MHUV2_STAT_BYTES))
 468				continue;
 469
 470			word = data[i];
 471			bytes_to_send = bytes_in_round & (MHUV2_STAT_BYTES - 1);
 472			if (unlikely(bytes_to_send))
 473				word &= LSB_MASK(bytes_to_send);
 474			else
 475				bytes_to_send = MHUV2_STAT_BYTES;
 476
 477			writel_relaxed(word, &mhu->send->ch_wn[priv->ch_wn_idx + windows - 1 - i].stat_set);
 478			bytes_left -= bytes_to_send;
 479			bytes_in_round -= bytes_to_send;
 480		}
 481
 482		data += windows;
 483	}
 484
 485	return 0;
 486}
 487
 488static const struct mhuv2_protocol_ops mhuv2_data_transfer_ops = {
 489	.rx_startup = mhuv2_data_transfer_rx_startup,
 490	.rx_shutdown = mhuv2_data_transfer_rx_shutdown,
 491	.read_data = mhuv2_data_transfer_read_data,
 492	.tx_startup = mhuv2_data_transfer_tx_startup,
 493	.tx_shutdown = mhuv2_data_transfer_tx_shutdown,
 494	.last_tx_done = mhuv2_data_transfer_last_tx_done,
 495	.send_data = mhuv2_data_transfer_send_data,
 496};
 497
 498/* Interrupt handlers */
 499
 500static struct mbox_chan *get_irq_chan_comb(struct mhuv2 *mhu, u32 __iomem *reg)
 501{
 502	struct mbox_chan *chans = mhu->mbox.chans;
 503	int channel = 0, i, offset = 0, windows, protocol, ch_wn;
 504	u32 stat;
 505
 506	for (i = 0; i < MHUV2_CMB_INT_ST_REG_CNT; i++) {
 507		stat = readl_relaxed(reg + i);
 508		if (!stat)
 509			continue;
 510
 511		ch_wn = i * MHUV2_STAT_BITS + __builtin_ctz(stat);
 512
 513		for (i = 0; i < mhu->length; i += 2) {
 514			protocol = mhu->protocols[i];
 515			windows = mhu->protocols[i + 1];
 516
 517			if (ch_wn >= offset + windows) {
 518				if (protocol == DOORBELL)
 519					channel += MHUV2_STAT_BITS * windows;
 520				else
 521					channel++;
 522
 523				offset += windows;
 524				continue;
 525			}
 526
 527			/* Return first chan of the window in doorbell mode */
 528			if (protocol == DOORBELL)
 529				channel += MHUV2_STAT_BITS * (ch_wn - offset);
 530
 531			return &chans[channel];
 532		}
 533	}
 534
 535	return ERR_PTR(-EIO);
 536}
 537
 538static irqreturn_t mhuv2_sender_interrupt(int irq, void *data)
 539{
 540	struct mhuv2 *mhu = data;
 541	struct device *dev = mhu->mbox.dev;
 542	struct mhuv2_mbox_chan_priv *priv;
 543	struct mbox_chan *chan;
 544	unsigned long flags;
 545	int i, found = 0;
 546	u32 stat;
 547
 548	chan = get_irq_chan_comb(mhu, mhu->send->chcomb_int_st);
 549	if (IS_ERR(chan)) {
 550		dev_warn(dev, "Failed to find channel for the Tx interrupt\n");
 551		return IRQ_NONE;
 552	}
 553	priv = chan->con_priv;
 554
 555	if (!IS_PROTOCOL_DOORBELL(priv)) {
 556		for (i = 0; i < priv->windows; i++)
 557			writel_relaxed(1, &mhu->send->ch_wn[priv->ch_wn_idx + i].int_clr);
 558
 559		if (chan->cl) {
 560			mbox_chan_txdone(chan, 0);
 561			return IRQ_HANDLED;
 562		}
 563
 564		dev_warn(dev, "Tx interrupt Received on channel (%u) not currently attached to a mailbox client\n",
 565			 priv->ch_wn_idx);
 566		return IRQ_NONE;
 567	}
 568
 569	/* Clear the interrupt first, so we don't miss any doorbell later */
 570	writel_relaxed(1, &mhu->send->ch_wn[priv->ch_wn_idx].int_clr);
 571
 572	/*
 573	 * In Doorbell mode, make sure no new transitions happen while the
 574	 * interrupt handler is trying to find the finished doorbell tx
 575	 * operations, else we may think few of the transfers were complete
 576	 * before they actually were.
 577	 */
 578	spin_lock_irqsave(&mhu->doorbell_pending_lock, flags);
 579
 580	/*
 581	 * In case of doorbell mode, the first channel of the window is returned
 582	 * by get_irq_chan_comb(). Find all the pending channels here.
 583	 */
 584	stat = readl_relaxed(&mhu->send->ch_wn[priv->ch_wn_idx].stat);
 585
 586	for (i = 0; i < MHUV2_STAT_BITS; i++) {
 587		priv = chan[i].con_priv;
 588
 589		/* Find cases where pending was 1, but stat's bit is cleared */
 590		if (priv->pending ^ ((stat >> i) & 0x1)) {
 591			BUG_ON(!priv->pending);
 592
 593			if (!chan->cl) {
 594				dev_warn(dev, "Tx interrupt received on doorbell (%u : %u) channel not currently attached to a mailbox client\n",
 595					 priv->ch_wn_idx, i);
 596				continue;
 597			}
 598
 599			mbox_chan_txdone(&chan[i], 0);
 600			priv->pending = 0;
 601			found++;
 602		}
 603	}
 604
 605	spin_unlock_irqrestore(&mhu->doorbell_pending_lock, flags);
 606
 607	if (!found) {
 608		/*
 609		 * We may have already processed the doorbell in the previous
 610		 * iteration if the interrupt came right after we cleared it but
 611		 * before we read the stat register.
 612		 */
 613		dev_dbg(dev, "Couldn't find the doorbell (%u) for the Tx interrupt interrupt\n",
 614			priv->ch_wn_idx);
 615		return IRQ_NONE;
 616	}
 617
 618	return IRQ_HANDLED;
 619}
 620
 621static struct mbox_chan *get_irq_chan_comb_rx(struct mhuv2 *mhu)
 622{
 623	struct mhuv2_mbox_chan_priv *priv;
 624	struct mbox_chan *chan;
 625	u32 stat;
 626
 627	chan = get_irq_chan_comb(mhu, mhu->recv->chcomb_int_st);
 628	if (IS_ERR(chan))
 629		return chan;
 630
 631	priv = chan->con_priv;
 632	if (!IS_PROTOCOL_DOORBELL(priv))
 633		return chan;
 634
 635	/*
 636	 * In case of doorbell mode, the first channel of the window is returned
 637	 * by the routine. Find the exact channel here.
 638	 */
 639	stat = readl_relaxed(&mhu->recv->ch_wn[priv->ch_wn_idx].stat_masked);
 640	BUG_ON(!stat);
 641
 642	return chan + __builtin_ctz(stat);
 643}
 644
 645static struct mbox_chan *get_irq_chan_stat_rx(struct mhuv2 *mhu)
 646{
 647	struct mbox_chan *chans = mhu->mbox.chans;
 648	struct mhuv2_mbox_chan_priv *priv;
 649	u32 stat;
 650	int i = 0;
 651
 652	while (i < mhu->mbox.num_chans) {
 653		priv = chans[i].con_priv;
 654		stat = readl_relaxed(&mhu->recv->ch_wn[priv->ch_wn_idx].stat_masked);
 655
 656		if (stat) {
 657			if (IS_PROTOCOL_DOORBELL(priv))
 658				i += __builtin_ctz(stat);
 659			return &chans[i];
 660		}
 661
 662		i += IS_PROTOCOL_DOORBELL(priv) ? MHUV2_STAT_BITS : 1;
 663	}
 664
 665	return ERR_PTR(-EIO);
 666}
 667
 668static struct mbox_chan *get_irq_chan_rx(struct mhuv2 *mhu)
 669{
 670	if (!mhu->minor)
 671		return get_irq_chan_stat_rx(mhu);
 672
 673	return get_irq_chan_comb_rx(mhu);
 674}
 675
 676static irqreturn_t mhuv2_receiver_interrupt(int irq, void *arg)
 677{
 678	struct mhuv2 *mhu = arg;
 679	struct mbox_chan *chan = get_irq_chan_rx(mhu);
 680	struct device *dev = mhu->mbox.dev;
 681	struct mhuv2_mbox_chan_priv *priv;
 682	int ret = IRQ_NONE;
 683	void *data;
 684
 685	if (IS_ERR(chan)) {
 686		dev_warn(dev, "Failed to find channel for the rx interrupt\n");
 687		return IRQ_NONE;
 688	}
 689	priv = chan->con_priv;
 690
 691	/* Read and clear the data first */
 692	data = priv->ops->read_data(mhu, chan);
 693
 694	if (!chan->cl) {
 695		dev_warn(dev, "Received data on channel (%u) not currently attached to a mailbox client\n",
 696			 priv->ch_wn_idx);
 697	} else if (IS_ERR(data)) {
 698		dev_err(dev, "Failed to read data: %lu\n", PTR_ERR(data));
 699	} else {
 700		mbox_chan_received_data(chan, data);
 701		ret = IRQ_HANDLED;
 702	}
 703
 704	if (!IS_ERR(data))
 705		kfree(data);
 706
 707	return ret;
 708}
 709
 710/* Sender and receiver ops */
 711static bool mhuv2_sender_last_tx_done(struct mbox_chan *chan)
 712{
 713	struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
 714	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 715
 716	return priv->ops->last_tx_done(mhu, chan);
 717}
 718
 719static int mhuv2_sender_send_data(struct mbox_chan *chan, void *data)
 720{
 721	struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
 722	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 723
 724	if (!priv->ops->last_tx_done(mhu, chan))
 725		return -EBUSY;
 726
 727	return priv->ops->send_data(mhu, chan, data);
 728}
 729
 730static int mhuv2_sender_startup(struct mbox_chan *chan)
 731{
 732	struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
 733	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 734
 735	if (priv->ops->tx_startup)
 736		priv->ops->tx_startup(mhu, chan);
 737	return 0;
 738}
 739
 740static void mhuv2_sender_shutdown(struct mbox_chan *chan)
 741{
 742	struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
 743	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 744
 745	if (priv->ops->tx_shutdown)
 746		priv->ops->tx_shutdown(mhu, chan);
 747}
 748
 749static const struct mbox_chan_ops mhuv2_sender_ops = {
 750	.send_data = mhuv2_sender_send_data,
 751	.startup = mhuv2_sender_startup,
 752	.shutdown = mhuv2_sender_shutdown,
 753	.last_tx_done = mhuv2_sender_last_tx_done,
 754};
 755
 756static int mhuv2_receiver_startup(struct mbox_chan *chan)
 757{
 758	struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
 759	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 760
 761	return priv->ops->rx_startup(mhu, chan);
 762}
 763
 764static void mhuv2_receiver_shutdown(struct mbox_chan *chan)
 765{
 766	struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
 767	struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
 768
 769	priv->ops->rx_shutdown(mhu, chan);
 770}
 771
 772static int mhuv2_receiver_send_data(struct mbox_chan *chan, void *data)
 773{
 774	dev_err(chan->mbox->dev,
 775		"Trying to transmit on a receiver MHU frame\n");
 776	return -EIO;
 777}
 778
 779static bool mhuv2_receiver_last_tx_done(struct mbox_chan *chan)
 780{
 781	dev_err(chan->mbox->dev, "Trying to Tx poll on a receiver MHU frame\n");
 782	return true;
 783}
 784
 785static const struct mbox_chan_ops mhuv2_receiver_ops = {
 786	.send_data = mhuv2_receiver_send_data,
 787	.startup = mhuv2_receiver_startup,
 788	.shutdown = mhuv2_receiver_shutdown,
 789	.last_tx_done = mhuv2_receiver_last_tx_done,
 790};
 791
 792static struct mbox_chan *mhuv2_mbox_of_xlate(struct mbox_controller *mbox,
 793					     const struct of_phandle_args *pa)
 794{
 795	struct mhuv2 *mhu = mhu_from_mbox(mbox);
 796	struct mbox_chan *chans = mbox->chans;
 797	int channel = 0, i, offset, doorbell, protocol, windows;
 798
 799	if (pa->args_count != 2)
 800		return ERR_PTR(-EINVAL);
 801
 802	offset = pa->args[0];
 803	doorbell = pa->args[1];
 804	if (doorbell >= MHUV2_STAT_BITS)
 805		goto out;
 806
 807	for (i = 0; i < mhu->length; i += 2) {
 808		protocol = mhu->protocols[i];
 809		windows = mhu->protocols[i + 1];
 810
 811		if (protocol == DOORBELL) {
 812			if (offset < windows)
 813				return &chans[channel + MHUV2_STAT_BITS * offset + doorbell];
 814
 815			channel += MHUV2_STAT_BITS * windows;
 816			offset -= windows;
 817		} else {
 818			if (offset == 0) {
 819				if (doorbell)
 820					goto out;
 821
 822				return &chans[channel];
 823			}
 824
 825			channel++;
 826			offset--;
 827		}
 828	}
 829
 830out:
 831	dev_err(mbox->dev, "Couldn't xlate to a valid channel (%d: %d)\n",
 832		pa->args[0], doorbell);
 833	return ERR_PTR(-ENODEV);
 834}
 835
 836static int mhuv2_verify_protocol(struct mhuv2 *mhu)
 837{
 838	struct device *dev = mhu->mbox.dev;
 839	int protocol, windows, channels = 0, total_windows = 0, i;
 840
 841	for (i = 0; i < mhu->length; i += 2) {
 842		protocol = mhu->protocols[i];
 843		windows = mhu->protocols[i + 1];
 844
 845		if (!windows) {
 846			dev_err(dev, "Window size can't be zero (%d)\n", i);
 847			return -EINVAL;
 848		}
 849		total_windows += windows;
 850
 851		if (protocol == DOORBELL) {
 852			channels += MHUV2_STAT_BITS * windows;
 853		} else if (protocol == DATA_TRANSFER) {
 854			channels++;
 855		} else {
 856			dev_err(dev, "Invalid protocol (%d) present in %s property at index %d\n",
 857				protocol, MHUV2_PROTOCOL_PROP, i);
 858			return -EINVAL;
 859		}
 860	}
 861
 862	if (total_windows > mhu->windows) {
 863		dev_err(dev, "Channel windows can't be more than what's implemented by the hardware ( %d: %d)\n",
 864			total_windows, mhu->windows);
 865		return -EINVAL;
 866	}
 867
 868	mhu->mbox.num_chans = channels;
 869	return 0;
 870}
 871
 872static int mhuv2_allocate_channels(struct mhuv2 *mhu)
 873{
 874	struct mbox_controller *mbox = &mhu->mbox;
 875	struct mhuv2_mbox_chan_priv *priv;
 876	struct device *dev = mbox->dev;
 877	struct mbox_chan *chans;
 878	int protocol, windows = 0, next_window = 0, i, j, k;
 879
 880	chans = devm_kcalloc(dev, mbox->num_chans, sizeof(*chans), GFP_KERNEL);
 881	if (!chans)
 882		return -ENOMEM;
 883
 884	mbox->chans = chans;
 885
 886	for (i = 0; i < mhu->length; i += 2) {
 887		next_window += windows;
 888
 889		protocol = mhu->protocols[i];
 890		windows = mhu->protocols[i + 1];
 891
 892		if (protocol == DATA_TRANSFER) {
 893			priv = devm_kmalloc(dev, sizeof(*priv), GFP_KERNEL);
 894			if (!priv)
 895				return -ENOMEM;
 896
 897			priv->ch_wn_idx = next_window;
 898			priv->ops = &mhuv2_data_transfer_ops;
 899			priv->windows = windows;
 900			chans++->con_priv = priv;
 901			continue;
 902		}
 903
 904		for (j = 0; j < windows; j++) {
 905			for (k = 0; k < MHUV2_STAT_BITS; k++) {
 906				priv = devm_kmalloc(dev, sizeof(*priv), GFP_KERNEL);
 907				if (!priv)
 908					return -ENOMEM;
 909
 910				priv->ch_wn_idx = next_window + j;
 911				priv->ops = &mhuv2_doorbell_ops;
 912				priv->doorbell = k;
 913				chans++->con_priv = priv;
 914			}
 915
 916			/*
 917			 * Permanently enable interrupt as we can't
 918			 * control it per doorbell.
 919			 */
 920			if (mhu->frame == SENDER_FRAME && mhu->minor)
 921				writel_relaxed(0x1, &mhu->send->ch_wn[priv->ch_wn_idx].int_en);
 922		}
 923	}
 924
 925	/* Make sure we have initialized all channels */
 926	BUG_ON(chans - mbox->chans != mbox->num_chans);
 927
 928	return 0;
 929}
 930
 931static int mhuv2_parse_channels(struct mhuv2 *mhu)
 932{
 933	struct device *dev = mhu->mbox.dev;
 934	const struct device_node *np = dev->of_node;
 935	int ret, count;
 936	u32 *protocols;
 937
 938	count = of_property_count_u32_elems(np, MHUV2_PROTOCOL_PROP);
 939	if (count <= 0 || count % 2) {
 940		dev_err(dev, "Invalid %s property (%d)\n", MHUV2_PROTOCOL_PROP,
 941			count);
 942		return -EINVAL;
 943	}
 944
 945	protocols = devm_kmalloc_array(dev, count, sizeof(*protocols), GFP_KERNEL);
 946	if (!protocols)
 947		return -ENOMEM;
 948
 949	ret = of_property_read_u32_array(np, MHUV2_PROTOCOL_PROP, protocols, count);
 950	if (ret) {
 951		dev_err(dev, "Failed to read %s property: %d\n",
 952			MHUV2_PROTOCOL_PROP, ret);
 953		return ret;
 954	}
 955
 956	mhu->protocols = protocols;
 957	mhu->length = count;
 958
 959	ret = mhuv2_verify_protocol(mhu);
 960	if (ret)
 961		return ret;
 962
 963	return mhuv2_allocate_channels(mhu);
 964}
 965
 966static int mhuv2_tx_init(struct amba_device *adev, struct mhuv2 *mhu,
 967			 void __iomem *reg)
 968{
 969	struct device *dev = mhu->mbox.dev;
 970	int ret, i;
 971
 972	mhu->frame = SENDER_FRAME;
 973	mhu->mbox.ops = &mhuv2_sender_ops;
 974	mhu->send = reg;
 975
 976	mhu->windows = readl_relaxed_bitfield(&mhu->send->mhu_cfg, struct mhu_cfg_t, num_ch);
 977	mhu->minor = readl_relaxed_bitfield(&mhu->send->aidr, struct aidr_t, arch_minor_rev);
 978
 979	spin_lock_init(&mhu->doorbell_pending_lock);
 980
 981	/*
 982	 * For minor version 1 and forward, tx interrupt is provided by
 983	 * the controller.
 984	 */
 985	if (mhu->minor && adev->irq[0]) {
 986		ret = devm_request_threaded_irq(dev, adev->irq[0], NULL,
 987						mhuv2_sender_interrupt,
 988						IRQF_ONESHOT, "mhuv2-tx", mhu);
 989		if (ret) {
 990			dev_err(dev, "Failed to request tx IRQ, fallback to polling mode: %d\n",
 991				ret);
 992		} else {
 993			mhu->mbox.txdone_irq = true;
 994			mhu->mbox.txdone_poll = false;
 995			mhu->irq = adev->irq[0];
 996
 997			writel_relaxed_bitfield(1, &mhu->send->int_en, struct int_en_t, chcomb);
 998
 999			/* Disable all channel interrupts */
1000			for (i = 0; i < mhu->windows; i++)
1001				writel_relaxed(0x0, &mhu->send->ch_wn[i].int_en);
1002
1003			goto out;
1004		}
1005	}
1006
1007	mhu->mbox.txdone_irq = false;
1008	mhu->mbox.txdone_poll = true;
1009	mhu->mbox.txpoll_period = 1;
1010
1011out:
1012	/* Wait for receiver to be ready */
1013	writel_relaxed(0x1, &mhu->send->access_request);
1014	while (!readl_relaxed(&mhu->send->access_ready))
1015		continue;
1016
1017	return 0;
1018}
1019
1020static int mhuv2_rx_init(struct amba_device *adev, struct mhuv2 *mhu,
1021			 void __iomem *reg)
1022{
1023	struct device *dev = mhu->mbox.dev;
1024	int ret, i;
1025
1026	mhu->frame = RECEIVER_FRAME;
1027	mhu->mbox.ops = &mhuv2_receiver_ops;
1028	mhu->recv = reg;
1029
1030	mhu->windows = readl_relaxed_bitfield(&mhu->recv->mhu_cfg, struct mhu_cfg_t, num_ch);
1031	mhu->minor = readl_relaxed_bitfield(&mhu->recv->aidr, struct aidr_t, arch_minor_rev);
1032
1033	mhu->irq = adev->irq[0];
1034	if (!mhu->irq) {
1035		dev_err(dev, "Missing receiver IRQ\n");
1036		return -EINVAL;
1037	}
1038
1039	ret = devm_request_threaded_irq(dev, mhu->irq, NULL,
1040					mhuv2_receiver_interrupt, IRQF_ONESHOT,
1041					"mhuv2-rx", mhu);
1042	if (ret) {
1043		dev_err(dev, "Failed to request rx IRQ\n");
1044		return ret;
1045	}
1046
1047	/* Mask all the channel windows */
1048	for (i = 0; i < mhu->windows; i++)
1049		writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_set);
1050
1051	if (mhu->minor)
1052		writel_relaxed_bitfield(1, &mhu->recv->int_en, struct int_en_t, chcomb);
1053
1054	return 0;
1055}
1056
1057static int mhuv2_probe(struct amba_device *adev, const struct amba_id *id)
1058{
1059	struct device *dev = &adev->dev;
1060	const struct device_node *np = dev->of_node;
1061	struct mhuv2 *mhu;
1062	void __iomem *reg;
1063	int ret = -EINVAL;
1064
1065	reg = devm_of_iomap(dev, dev->of_node, 0, NULL);
1066	if (IS_ERR(reg))
1067		return PTR_ERR(reg);
1068
1069	mhu = devm_kzalloc(dev, sizeof(*mhu), GFP_KERNEL);
1070	if (!mhu)
1071		return -ENOMEM;
1072
1073	mhu->mbox.dev = dev;
1074	mhu->mbox.of_xlate = mhuv2_mbox_of_xlate;
1075
1076	if (of_device_is_compatible(np, "arm,mhuv2-tx"))
1077		ret = mhuv2_tx_init(adev, mhu, reg);
1078	else if (of_device_is_compatible(np, "arm,mhuv2-rx"))
1079		ret = mhuv2_rx_init(adev, mhu, reg);
1080	else
1081		dev_err(dev, "Invalid compatible property\n");
1082
1083	if (ret)
1084		return ret;
1085
1086	/* Channel windows can't be 0 */
1087	BUG_ON(!mhu->windows);
1088
1089	ret = mhuv2_parse_channels(mhu);
1090	if (ret)
1091		return ret;
1092
1093	amba_set_drvdata(adev, mhu);
1094
1095	ret = devm_mbox_controller_register(dev, &mhu->mbox);
1096	if (ret)
1097		dev_err(dev, "failed to register ARM MHUv2 driver %d\n", ret);
1098
1099	return ret;
1100}
1101
1102static void mhuv2_remove(struct amba_device *adev)
1103{
1104	struct mhuv2 *mhu = amba_get_drvdata(adev);
1105
1106	if (mhu->frame == SENDER_FRAME)
1107		writel_relaxed(0x0, &mhu->send->access_request);
1108}
1109
1110static struct amba_id mhuv2_ids[] = {
1111	{
1112		/* 2.0 */
1113		.id = 0xbb0d1,
1114		.mask = 0xfffff,
1115	},
1116	{
1117		/* 2.1 */
1118		.id = 0xbb076,
1119		.mask = 0xfffff,
1120	},
1121	{ 0, 0 },
1122};
1123MODULE_DEVICE_TABLE(amba, mhuv2_ids);
1124
1125static struct amba_driver mhuv2_driver = {
1126	.drv = {
1127		.name	= "arm-mhuv2",
1128	},
1129	.id_table	= mhuv2_ids,
1130	.probe		= mhuv2_probe,
1131	.remove		= mhuv2_remove,
1132};
1133module_amba_driver(mhuv2_driver);
1134
1135MODULE_LICENSE("GPL v2");
1136MODULE_DESCRIPTION("ARM MHUv2 Driver");
1137MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>");
1138MODULE_AUTHOR("Tushar Khandelwal <tushar.khandelwal@arm.com>");