1/*
   2 * Copyright (C) 2017 Broadcom
   3 *
   4 * This program is free software; you can redistribute it and/or
   5 * modify it under the terms of the GNU General Public License as
   6 * published by the Free Software Foundation version 2.
   7 *
   8 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
   9 * kind, whether express or implied; without even the implied warranty
  10 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 * GNU General Public License for more details.
  12 */
  13
  14/*
  15 * Broadcom FlexRM Mailbox Driver
  16 *
  17 * Each Broadcom FlexSparx4 offload engine is implemented as an
  18 * extension to Broadcom FlexRM ring manager. The FlexRM ring
  19 * manager provides a set of rings which can be used to submit
  20 * work to a FlexSparx4 offload engine.
  21 *
  22 * This driver creates a mailbox controller using a set of FlexRM
  23 * rings where each mailbox channel represents a separate FlexRM ring.
  24 */
  25
  26#include <asm/barrier.h>
  27#include <asm/byteorder.h>
  28#include <linux/atomic.h>
  29#include <linux/bitmap.h>
  30#include <linux/debugfs.h>
  31#include <linux/delay.h>
  32#include <linux/device.h>
  33#include <linux/dma-mapping.h>
  34#include <linux/dmapool.h>
  35#include <linux/err.h>
  36#include <linux/interrupt.h>
  37#include <linux/kernel.h>
  38#include <linux/mailbox_controller.h>
  39#include <linux/mailbox_client.h>
  40#include <linux/mailbox/brcm-message.h>
  41#include <linux/module.h>
  42#include <linux/msi.h>
  43#include <linux/of_address.h>
  44#include <linux/of_irq.h>
  45#include <linux/platform_device.h>
  46#include <linux/spinlock.h>
  47
  48/* ====== FlexRM register defines ===== */
  49
  50/* FlexRM configuration */
  51#define RING_REGS_SIZE					0x10000
  52#define RING_DESC_SIZE					8
  53#define RING_DESC_INDEX(offset)				\
  54			((offset) / RING_DESC_SIZE)
  55#define RING_DESC_OFFSET(index)				\
  56			((index) * RING_DESC_SIZE)
  57#define RING_MAX_REQ_COUNT				1024
  58#define RING_BD_ALIGN_ORDER				12
  59#define RING_BD_ALIGN_CHECK(addr)			\
  60			(!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))
  61#define RING_BD_TOGGLE_INVALID(offset)			\
  62			(((offset) >> RING_BD_ALIGN_ORDER) & 0x1)
  63#define RING_BD_TOGGLE_VALID(offset)			\
  64			(!RING_BD_TOGGLE_INVALID(offset))
  65#define RING_BD_DESC_PER_REQ				32
  66#define RING_BD_DESC_COUNT				\
  67			(RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)
  68#define RING_BD_SIZE					\
  69			(RING_BD_DESC_COUNT * RING_DESC_SIZE)
  70#define RING_CMPL_ALIGN_ORDER				13
  71#define RING_CMPL_DESC_COUNT				RING_MAX_REQ_COUNT
  72#define RING_CMPL_SIZE					\
  73			(RING_CMPL_DESC_COUNT * RING_DESC_SIZE)
  74#define RING_VER_MAGIC					0x76303031
  75
  76/* Per-Ring register offsets */
  77#define RING_VER					0x000
  78#define RING_BD_START_ADDR				0x004
  79#define RING_BD_READ_PTR				0x008
  80#define RING_BD_WRITE_PTR				0x00c
  81#define RING_BD_READ_PTR_DDR_LS				0x010
  82#define RING_BD_READ_PTR_DDR_MS				0x014
  83#define RING_CMPL_START_ADDR				0x018
  84#define RING_CMPL_WRITE_PTR				0x01c
  85#define RING_NUM_REQ_RECV_LS				0x020
  86#define RING_NUM_REQ_RECV_MS				0x024
  87#define RING_NUM_REQ_TRANS_LS				0x028
  88#define RING_NUM_REQ_TRANS_MS				0x02c
  89#define RING_NUM_REQ_OUTSTAND				0x030
  90#define RING_CONTROL					0x034
  91#define RING_FLUSH_DONE					0x038
  92#define RING_MSI_ADDR_LS				0x03c
  93#define RING_MSI_ADDR_MS				0x040
  94#define RING_MSI_CONTROL				0x048
  95#define RING_BD_READ_PTR_DDR_CONTROL			0x04c
  96#define RING_MSI_DATA_VALUE				0x064
  97
  98/* Register RING_BD_START_ADDR fields */
  99#define BD_LAST_UPDATE_HW_SHIFT				28
 100#define BD_LAST_UPDATE_HW_MASK				0x1
 101#define BD_START_ADDR_VALUE(pa)				\
 102	((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
 103#define BD_START_ADDR_DECODE(val)			\
 104	((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
 105
 106/* Register RING_CMPL_START_ADDR fields */
 107#define CMPL_START_ADDR_VALUE(pa)			\
 108	((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
 109
 110/* Register RING_CONTROL fields */
 111#define CONTROL_MASK_DISABLE_CONTROL			12
 112#define CONTROL_FLUSH_SHIFT				5
 113#define CONTROL_ACTIVE_SHIFT				4
 114#define CONTROL_RATE_ADAPT_MASK				0xf
 115#define CONTROL_RATE_DYNAMIC				0x0
 116#define CONTROL_RATE_FAST				0x8
 117#define CONTROL_RATE_MEDIUM				0x9
 118#define CONTROL_RATE_SLOW				0xa
 119#define CONTROL_RATE_IDLE				0xb
 120
 121/* Register RING_FLUSH_DONE fields */
 122#define FLUSH_DONE_MASK					0x1
 123
 124/* Register RING_MSI_CONTROL fields */
 125#define MSI_TIMER_VAL_SHIFT				16
 126#define MSI_TIMER_VAL_MASK				0xffff
 127#define MSI_ENABLE_SHIFT				15
 128#define MSI_ENABLE_MASK					0x1
 129#define MSI_COUNT_SHIFT					0
 130#define MSI_COUNT_MASK					0x3ff
 131
 132/* Register RING_BD_READ_PTR_DDR_CONTROL fields */
 133#define BD_READ_PTR_DDR_TIMER_VAL_SHIFT			16
 134#define BD_READ_PTR_DDR_TIMER_VAL_MASK			0xffff
 135#define BD_READ_PTR_DDR_ENABLE_SHIFT			15
 136#define BD_READ_PTR_DDR_ENABLE_MASK			0x1
 137
 138/* ====== FlexRM ring descriptor defines ===== */
 139
 140/* Completion descriptor format */
 141#define CMPL_OPAQUE_SHIFT			0
 142#define CMPL_OPAQUE_MASK			0xffff
 143#define CMPL_ENGINE_STATUS_SHIFT		16
 144#define CMPL_ENGINE_STATUS_MASK			0xffff
 145#define CMPL_DME_STATUS_SHIFT			32
 146#define CMPL_DME_STATUS_MASK			0xffff
 147#define CMPL_RM_STATUS_SHIFT			48
 148#define CMPL_RM_STATUS_MASK			0xffff
 149
 150/* Completion DME status code */
 151#define DME_STATUS_MEM_COR_ERR			BIT(0)
 152#define DME_STATUS_MEM_UCOR_ERR			BIT(1)
 153#define DME_STATUS_FIFO_UNDERFLOW		BIT(2)
 154#define DME_STATUS_FIFO_OVERFLOW		BIT(3)
 155#define DME_STATUS_RRESP_ERR			BIT(4)
 156#define DME_STATUS_BRESP_ERR			BIT(5)
 157#define DME_STATUS_ERROR_MASK			(DME_STATUS_MEM_COR_ERR | \
 158						 DME_STATUS_MEM_UCOR_ERR | \
 159						 DME_STATUS_FIFO_UNDERFLOW | \
 160						 DME_STATUS_FIFO_OVERFLOW | \
 161						 DME_STATUS_RRESP_ERR | \
 162						 DME_STATUS_BRESP_ERR)
 163
 164/* Completion RM status code */
 165#define RM_STATUS_CODE_SHIFT			0
 166#define RM_STATUS_CODE_MASK			0x3ff
 167#define RM_STATUS_CODE_GOOD			0x0
 168#define RM_STATUS_CODE_AE_TIMEOUT		0x3ff
 169
 170/* General descriptor format */
 171#define DESC_TYPE_SHIFT				60
 172#define DESC_TYPE_MASK				0xf
 173#define DESC_PAYLOAD_SHIFT			0
 174#define DESC_PAYLOAD_MASK			0x0fffffffffffffff
 175
 176/* Null descriptor format  */
 177#define NULL_TYPE				0
 178#define NULL_TOGGLE_SHIFT			58
 179#define NULL_TOGGLE_MASK			0x1
 180
 181/* Header descriptor format */
 182#define HEADER_TYPE				1
 183#define HEADER_TOGGLE_SHIFT			58
 184#define HEADER_TOGGLE_MASK			0x1
 185#define HEADER_ENDPKT_SHIFT			57
 186#define HEADER_ENDPKT_MASK			0x1
 187#define HEADER_STARTPKT_SHIFT			56
 188#define HEADER_STARTPKT_MASK			0x1
 189#define HEADER_BDCOUNT_SHIFT			36
 190#define HEADER_BDCOUNT_MASK			0x1f
 191#define HEADER_BDCOUNT_MAX			HEADER_BDCOUNT_MASK
 192#define HEADER_FLAGS_SHIFT			16
 193#define HEADER_FLAGS_MASK			0xffff
 194#define HEADER_OPAQUE_SHIFT			0
 195#define HEADER_OPAQUE_MASK			0xffff
 196
 197/* Source (SRC) descriptor format */
 198#define SRC_TYPE				2
 199#define SRC_LENGTH_SHIFT			44
 200#define SRC_LENGTH_MASK				0xffff
 201#define SRC_ADDR_SHIFT				0
 202#define SRC_ADDR_MASK				0x00000fffffffffff
 203
 204/* Destination (DST) descriptor format */
 205#define DST_TYPE				3
 206#define DST_LENGTH_SHIFT			44
 207#define DST_LENGTH_MASK				0xffff
 208#define DST_ADDR_SHIFT				0
 209#define DST_ADDR_MASK				0x00000fffffffffff
 210
 211/* Immediate (IMM) descriptor format */
 212#define IMM_TYPE				4
 213#define IMM_DATA_SHIFT				0
 214#define IMM_DATA_MASK				0x0fffffffffffffff
 215
 216/* Next pointer (NPTR) descriptor format */
 217#define NPTR_TYPE				5
 218#define NPTR_TOGGLE_SHIFT			58
 219#define NPTR_TOGGLE_MASK			0x1
 220#define NPTR_ADDR_SHIFT				0
 221#define NPTR_ADDR_MASK				0x00000fffffffffff
 222
 223/* Mega source (MSRC) descriptor format */
 224#define MSRC_TYPE				6
 225#define MSRC_LENGTH_SHIFT			44
 226#define MSRC_LENGTH_MASK			0xffff
 227#define MSRC_ADDR_SHIFT				0
 228#define MSRC_ADDR_MASK				0x00000fffffffffff
 229
 230/* Mega destination (MDST) descriptor format */
 231#define MDST_TYPE				7
 232#define MDST_LENGTH_SHIFT			44
 233#define MDST_LENGTH_MASK			0xffff
 234#define MDST_ADDR_SHIFT				0
 235#define MDST_ADDR_MASK				0x00000fffffffffff
 236
 237/* Source with tlast (SRCT) descriptor format */
 238#define SRCT_TYPE				8
 239#define SRCT_LENGTH_SHIFT			44
 240#define SRCT_LENGTH_MASK			0xffff
 241#define SRCT_ADDR_SHIFT				0
 242#define SRCT_ADDR_MASK				0x00000fffffffffff
 243
 244/* Destination with tlast (DSTT) descriptor format */
 245#define DSTT_TYPE				9
 246#define DSTT_LENGTH_SHIFT			44
 247#define DSTT_LENGTH_MASK			0xffff
 248#define DSTT_ADDR_SHIFT				0
 249#define DSTT_ADDR_MASK				0x00000fffffffffff
 250
 251/* Immediate with tlast (IMMT) descriptor format */
 252#define IMMT_TYPE				10
 253#define IMMT_DATA_SHIFT				0
 254#define IMMT_DATA_MASK				0x0fffffffffffffff
 255
 256/* Descriptor helper macros */
 257#define DESC_DEC(_d, _s, _m)			(((_d) >> (_s)) & (_m))
 258#define DESC_ENC(_d, _v, _s, _m)		\
 259			do { \
 260				(_d) &= ~((u64)(_m) << (_s)); \
 261				(_d) |= (((u64)(_v) & (_m)) << (_s)); \
 262			} while (0)
 263
 264/* ====== FlexRM data structures ===== */
 265
 266struct flexrm_ring {
 267	/* Unprotected members */
 268	int num;
 269	struct flexrm_mbox *mbox;
 270	void __iomem *regs;
 271	bool irq_requested;
 272	unsigned int irq;
 273	cpumask_t irq_aff_hint;
 274	unsigned int msi_timer_val;
 275	unsigned int msi_count_threshold;
 276	struct brcm_message *requests[RING_MAX_REQ_COUNT];
 277	void *bd_base;
 278	dma_addr_t bd_dma_base;
 279	u32 bd_write_offset;
 280	void *cmpl_base;
 281	dma_addr_t cmpl_dma_base;
 282	/* Atomic stats */
 283	atomic_t msg_send_count;
 284	atomic_t msg_cmpl_count;
 285	/* Protected members */
 286	spinlock_t lock;
 287	DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
 288	u32 cmpl_read_offset;
 289};
 290
 291struct flexrm_mbox {
 292	struct device *dev;
 293	void __iomem *regs;
 294	u32 num_rings;
 295	struct flexrm_ring *rings;
 296	struct dma_pool *bd_pool;
 297	struct dma_pool *cmpl_pool;
 298	struct dentry *root;
 299	struct mbox_controller controller;
 300};
 301
 302/* ====== FlexRM ring descriptor helper routines ===== */
 303
 304static u64 flexrm_read_desc(void *desc_ptr)
 305{
 306	return le64_to_cpu(*((u64 *)desc_ptr));
 307}
 308
 309static void flexrm_write_desc(void *desc_ptr, u64 desc)
 310{
 311	*((u64 *)desc_ptr) = cpu_to_le64(desc);
 312}
 313
 314static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)
 315{
 316	return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);
 317}
 318
 319static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)
 320{
 321	u32 status;
 322
 323	status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,
 324			  CMPL_DME_STATUS_MASK);
 325	if (status & DME_STATUS_ERROR_MASK)
 326		return -EIO;
 327
 328	status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,
 329			  CMPL_RM_STATUS_MASK);
 330	status &= RM_STATUS_CODE_MASK;
 331	if (status == RM_STATUS_CODE_AE_TIMEOUT)
 332		return -ETIMEDOUT;
 333
 334	return 0;
 335}
 336
 337static bool flexrm_is_next_table_desc(void *desc_ptr)
 338{
 339	u64 desc = flexrm_read_desc(desc_ptr);
 340	u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 341
 342	return (type == NPTR_TYPE) ? true : false;
 343}
 344
 345static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)
 346{
 347	u64 desc = 0;
 348
 349	DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 350	DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);
 351	DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);
 352
 353	return desc;
 354}
 355
 356static u64 flexrm_null_desc(u32 toggle)
 357{
 358	u64 desc = 0;
 359
 360	DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 361	DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);
 362
 363	return desc;
 364}
 365
 366static u32 flexrm_estimate_header_desc_count(u32 nhcnt)
 367{
 368	u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;
 369
 370	if (!(nhcnt % HEADER_BDCOUNT_MAX))
 371		hcnt += 1;
 372
 373	return hcnt;
 374}
 375
 376static void flexrm_flip_header_toggle(void *desc_ptr)
 377{
 378	u64 desc = flexrm_read_desc(desc_ptr);
 379
 380	if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))
 381		desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);
 382	else
 383		desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);
 384
 385	flexrm_write_desc(desc_ptr, desc);
 386}
 387
 388static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,
 389			       u32 bdcount, u32 flags, u32 opaque)
 390{
 391	u64 desc = 0;
 392
 393	DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 394	DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);
 395	DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);
 396	DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);
 397	DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);
 398	DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);
 399	DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);
 400
 401	return desc;
 402}
 403
 404static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,
 405				 u64 desc, void **desc_ptr, u32 *toggle,
 406				 void *start_desc, void *end_desc)
 407{
 408	u64 d;
 409	u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;
 410
 411	/* Sanity check */
 412	if (nhcnt <= nhpos)
 413		return;
 414
 415	/*
 416	 * Each request or packet start with a HEADER descriptor followed
 417	 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
 418	 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
 419	 * following a HEADER descriptor is represented by BDCOUNT field
 420	 * of HEADER descriptor. The max value of BDCOUNT field is 31 which
 421	 * means we can only have 31 non-HEADER descriptors following one
 422	 * HEADER descriptor.
 423	 *
 424	 * In general use, number of non-HEADER descriptors can easily go
 425	 * beyond 31. To tackle this situation, we have packet (or request)
 426	 * extenstion bits (STARTPKT and ENDPKT) in the HEADER descriptor.
 427	 *
 428	 * To use packet extension, the first HEADER descriptor of request
 429	 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
 430	 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
 431	 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
 432	 * TOGGLE bit of the first HEADER will be set to invalid state to
 433	 * ensure that FlexRM does not start fetching descriptors till all
 434	 * descriptors are enqueued. The user of this function will flip
 435	 * the TOGGLE bit of first HEADER after all descriptors are
 436	 * enqueued.
 437	 */
 438
 439	if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
 440		/* Prepare the header descriptor */
 441		nhavail = (nhcnt - nhpos);
 442		_toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
 443		_startpkt = (nhpos == 0) ? 0x1 : 0x0;
 444		_endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
 445		_bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
 446				nhavail : HEADER_BDCOUNT_MAX;
 447		if (nhavail <= HEADER_BDCOUNT_MAX)
 448			_bdcount = nhavail;
 449		else
 450			_bdcount = HEADER_BDCOUNT_MAX;
 451		d = flexrm_header_desc(_toggle, _startpkt, _endpkt,
 452					_bdcount, 0x0, reqid);
 453
 454		/* Write header descriptor */
 455		flexrm_write_desc(*desc_ptr, d);
 456
 457		/* Point to next descriptor */
 458		*desc_ptr += sizeof(desc);
 459		if (*desc_ptr == end_desc)
 460			*desc_ptr = start_desc;
 461
 462		/* Skip next pointer descriptors */
 463		while (flexrm_is_next_table_desc(*desc_ptr)) {
 464			*toggle = (*toggle) ? 0 : 1;
 465			*desc_ptr += sizeof(desc);
 466			if (*desc_ptr == end_desc)
 467				*desc_ptr = start_desc;
 468		}
 469	}
 470
 471	/* Write desired descriptor */
 472	flexrm_write_desc(*desc_ptr, desc);
 473
 474	/* Point to next descriptor */
 475	*desc_ptr += sizeof(desc);
 476	if (*desc_ptr == end_desc)
 477		*desc_ptr = start_desc;
 478
 479	/* Skip next pointer descriptors */
 480	while (flexrm_is_next_table_desc(*desc_ptr)) {
 481		*toggle = (*toggle) ? 0 : 1;
 482		*desc_ptr += sizeof(desc);
 483		if (*desc_ptr == end_desc)
 484			*desc_ptr = start_desc;
 485	}
 486}
 487
 488static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)
 489{
 490	u64 desc = 0;
 491
 492	DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 493	DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);
 494	DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);
 495
 496	return desc;
 497}
 498
 499static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)
 500{
 501	u64 desc = 0;
 502
 503	DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 504	DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);
 505	DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);
 506
 507	return desc;
 508}
 509
 510static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)
 511{
 512	u64 desc = 0;
 513
 514	DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 515	DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);
 516	DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);
 517
 518	return desc;
 519}
 520
 521static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)
 522{
 523	u64 desc = 0;
 524
 525	DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 526	DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);
 527	DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);
 528
 529	return desc;
 530}
 531
 532static u64 flexrm_imm_desc(u64 data)
 533{
 534	u64 desc = 0;
 535
 536	DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 537	DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);
 538
 539	return desc;
 540}
 541
 542static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)
 543{
 544	u64 desc = 0;
 545
 546	DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 547	DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);
 548	DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);
 549
 550	return desc;
 551}
 552
 553static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)
 554{
 555	u64 desc = 0;
 556
 557	DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 558	DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);
 559	DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);
 560
 561	return desc;
 562}
 563
 564static u64 flexrm_immt_desc(u64 data)
 565{
 566	u64 desc = 0;
 567
 568	DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 569	DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);
 570
 571	return desc;
 572}
 573
 574static bool flexrm_spu_sanity_check(struct brcm_message *msg)
 575{
 576	struct scatterlist *sg;
 577
 578	if (!msg->spu.src || !msg->spu.dst)
 579		return false;
 580	for (sg = msg->spu.src; sg; sg = sg_next(sg)) {
 581		if (sg->length & 0xf) {
 582			if (sg->length > SRC_LENGTH_MASK)
 583				return false;
 584		} else {
 585			if (sg->length > (MSRC_LENGTH_MASK * 16))
 586				return false;
 587		}
 588	}
 589	for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {
 590		if (sg->length & 0xf) {
 591			if (sg->length > DST_LENGTH_MASK)
 592				return false;
 593		} else {
 594			if (sg->length > (MDST_LENGTH_MASK * 16))
 595				return false;
 596		}
 597	}
 598
 599	return true;
 600}
 601
 602static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)
 603{
 604	u32 cnt = 0;
 605	unsigned int dst_target = 0;
 606	struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
 607
 608	while (src_sg || dst_sg) {
 609		if (src_sg) {
 610			cnt++;
 611			dst_target = src_sg->length;
 612			src_sg = sg_next(src_sg);
 613		} else
 614			dst_target = UINT_MAX;
 615
 616		while (dst_target && dst_sg) {
 617			cnt++;
 618			if (dst_sg->length < dst_target)
 619				dst_target -= dst_sg->length;
 620			else
 621				dst_target = 0;
 622			dst_sg = sg_next(dst_sg);
 623		}
 624	}
 625
 626	return cnt;
 627}
 628
 629static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)
 630{
 631	int rc;
 632
 633	rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
 634			DMA_TO_DEVICE);
 635	if (rc < 0)
 636		return rc;
 637
 638	rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
 639			DMA_FROM_DEVICE);
 640	if (rc < 0) {
 641		dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
 642			     DMA_TO_DEVICE);
 643		return rc;
 644	}
 645
 646	return 0;
 647}
 648
 649static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)
 650{
 651	dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
 652		     DMA_FROM_DEVICE);
 653	dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
 654		     DMA_TO_DEVICE);
 655}
 656
 657static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,
 658				     u32 reqid, void *desc_ptr, u32 toggle,
 659				     void *start_desc, void *end_desc)
 660{
 661	u64 d;
 662	u32 nhpos = 0;
 663	void *orig_desc_ptr = desc_ptr;
 664	unsigned int dst_target = 0;
 665	struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
 666
 667	while (src_sg || dst_sg) {
 668		if (src_sg) {
 669			if (sg_dma_len(src_sg) & 0xf)
 670				d = flexrm_src_desc(sg_dma_address(src_sg),
 671						     sg_dma_len(src_sg));
 672			else
 673				d = flexrm_msrc_desc(sg_dma_address(src_sg),
 674						      sg_dma_len(src_sg)/16);
 675			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 676					     d, &desc_ptr, &toggle,
 677					     start_desc, end_desc);
 678			nhpos++;
 679			dst_target = sg_dma_len(src_sg);
 680			src_sg = sg_next(src_sg);
 681		} else
 682			dst_target = UINT_MAX;
 683
 684		while (dst_target && dst_sg) {
 685			if (sg_dma_len(dst_sg) & 0xf)
 686				d = flexrm_dst_desc(sg_dma_address(dst_sg),
 687						     sg_dma_len(dst_sg));
 688			else
 689				d = flexrm_mdst_desc(sg_dma_address(dst_sg),
 690						      sg_dma_len(dst_sg)/16);
 691			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 692					     d, &desc_ptr, &toggle,
 693					     start_desc, end_desc);
 694			nhpos++;
 695			if (sg_dma_len(dst_sg) < dst_target)
 696				dst_target -= sg_dma_len(dst_sg);
 697			else
 698				dst_target = 0;
 699			dst_sg = sg_next(dst_sg);
 700		}
 701	}
 702
 703	/* Null descriptor with invalid toggle bit */
 704	flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
 705
 706	/* Ensure that descriptors have been written to memory */
 707	wmb();
 708
 709	/* Flip toggle bit in header */
 710	flexrm_flip_header_toggle(orig_desc_ptr);
 711
 712	return desc_ptr;
 713}
 714
 715static bool flexrm_sba_sanity_check(struct brcm_message *msg)
 716{
 717	u32 i;
 718
 719	if (!msg->sba.cmds || !msg->sba.cmds_count)
 720		return false;
 721
 722	for (i = 0; i < msg->sba.cmds_count; i++) {
 723		if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
 724		     (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&
 725		    (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))
 726			return false;
 727		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&
 728		    (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
 729			return false;
 730		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&
 731		    (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
 732			return false;
 733		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&
 734		    (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))
 735			return false;
 736		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&
 737		    (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))
 738			return false;
 739	}
 740
 741	return true;
 742}
 743
 744static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)
 745{
 746	u32 i, cnt;
 747
 748	cnt = 0;
 749	for (i = 0; i < msg->sba.cmds_count; i++) {
 750		cnt++;
 751
 752		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
 753		    (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))
 754			cnt++;
 755
 756		if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)
 757			cnt++;
 758
 759		if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)
 760			cnt++;
 761	}
 762
 763	return cnt;
 764}
 765
 766static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,
 767				     u32 reqid, void *desc_ptr, u32 toggle,
 768				     void *start_desc, void *end_desc)
 769{
 770	u64 d;
 771	u32 i, nhpos = 0;
 772	struct brcm_sba_command *c;
 773	void *orig_desc_ptr = desc_ptr;
 774
 775	/* Convert SBA commands into descriptors */
 776	for (i = 0; i < msg->sba.cmds_count; i++) {
 777		c = &msg->sba.cmds[i];
 778
 779		if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&
 780		    (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {
 781			/* Destination response descriptor */
 782			d = flexrm_dst_desc(c->resp, c->resp_len);
 783			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 784					     d, &desc_ptr, &toggle,
 785					     start_desc, end_desc);
 786			nhpos++;
 787		} else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {
 788			/* Destination response with tlast descriptor */
 789			d = flexrm_dstt_desc(c->resp, c->resp_len);
 790			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 791					     d, &desc_ptr, &toggle,
 792					     start_desc, end_desc);
 793			nhpos++;
 794		}
 795
 796		if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {
 797			/* Destination with tlast descriptor */
 798			d = flexrm_dstt_desc(c->data, c->data_len);
 799			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 800					     d, &desc_ptr, &toggle,
 801					     start_desc, end_desc);
 802			nhpos++;
 803		}
 804
 805		if (c->flags & BRCM_SBA_CMD_TYPE_B) {
 806			/* Command as immediate descriptor */
 807			d = flexrm_imm_desc(c->cmd);
 808			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 809					     d, &desc_ptr, &toggle,
 810					     start_desc, end_desc);
 811			nhpos++;
 812		} else {
 813			/* Command as immediate descriptor with tlast */
 814			d = flexrm_immt_desc(c->cmd);
 815			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 816					     d, &desc_ptr, &toggle,
 817					     start_desc, end_desc);
 818			nhpos++;
 819		}
 820
 821		if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||
 822		    (c->flags & BRCM_SBA_CMD_TYPE_C)) {
 823			/* Source with tlast descriptor */
 824			d = flexrm_srct_desc(c->data, c->data_len);
 825			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
 826					     d, &desc_ptr, &toggle,
 827					     start_desc, end_desc);
 828			nhpos++;
 829		}
 830	}
 831
 832	/* Null descriptor with invalid toggle bit */
 833	flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
 834
 835	/* Ensure that descriptors have been written to memory */
 836	wmb();
 837
 838	/* Flip toggle bit in header */
 839	flexrm_flip_header_toggle(orig_desc_ptr);
 840
 841	return desc_ptr;
 842}
 843
 844static bool flexrm_sanity_check(struct brcm_message *msg)
 845{
 846	if (!msg)
 847		return false;
 848
 849	switch (msg->type) {
 850	case BRCM_MESSAGE_SPU:
 851		return flexrm_spu_sanity_check(msg);
 852	case BRCM_MESSAGE_SBA:
 853		return flexrm_sba_sanity_check(msg);
 854	default:
 855		return false;
 856	};
 857}
 858
 859static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)
 860{
 861	if (!msg)
 862		return 0;
 863
 864	switch (msg->type) {
 865	case BRCM_MESSAGE_SPU:
 866		return flexrm_spu_estimate_nonheader_desc_count(msg);
 867	case BRCM_MESSAGE_SBA:
 868		return flexrm_sba_estimate_nonheader_desc_count(msg);
 869	default:
 870		return 0;
 871	};
 872}
 873
 874static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)
 875{
 876	if (!dev || !msg)
 877		return -EINVAL;
 878
 879	switch (msg->type) {
 880	case BRCM_MESSAGE_SPU:
 881		return flexrm_spu_dma_map(dev, msg);
 882	default:
 883		break;
 884	}
 885
 886	return 0;
 887}
 888
 889static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)
 890{
 891	if (!dev || !msg)
 892		return;
 893
 894	switch (msg->type) {
 895	case BRCM_MESSAGE_SPU:
 896		flexrm_spu_dma_unmap(dev, msg);
 897		break;
 898	default:
 899		break;
 900	}
 901}
 902
 903static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
 904				u32 reqid, void *desc_ptr, u32 toggle,
 905				void *start_desc, void *end_desc)
 906{
 907	if (!msg || !desc_ptr || !start_desc || !end_desc)
 908		return ERR_PTR(-ENOTSUPP);
 909
 910	if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))
 911		return ERR_PTR(-ERANGE);
 912
 913	switch (msg->type) {
 914	case BRCM_MESSAGE_SPU:
 915		return flexrm_spu_write_descs(msg, nhcnt, reqid,
 916					       desc_ptr, toggle,
 917					       start_desc, end_desc);
 918	case BRCM_MESSAGE_SBA:
 919		return flexrm_sba_write_descs(msg, nhcnt, reqid,
 920					       desc_ptr, toggle,
 921					       start_desc, end_desc);
 922	default:
 923		return ERR_PTR(-ENOTSUPP);
 924	};
 925}
 926
 927/* ====== FlexRM driver helper routines ===== */
 928
 929static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
 930					   struct seq_file *file)
 931{
 932	int i;
 933	const char *state;
 934	struct flexrm_ring *ring;
 935
 936	seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
 937		   "Ring#", "State", "BD_Addr", "BD_Size",
 938		   "Cmpl_Addr", "Cmpl_Size");
 939
 940	for (i = 0; i < mbox->num_rings; i++) {
 941		ring = &mbox->rings[i];
 942		if (readl(ring->regs + RING_CONTROL) &
 943		    BIT(CONTROL_ACTIVE_SHIFT))
 944			state = "active";
 945		else
 946			state = "inactive";
 947		seq_printf(file,
 948			   "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
 949			   ring->num, state,
 950			   (unsigned long long)ring->bd_dma_base,
 951			   (u32)RING_BD_SIZE,
 952			   (unsigned long long)ring->cmpl_dma_base,
 953			   (u32)RING_CMPL_SIZE);
 954	}
 955}
 956
 957static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
 958					  struct seq_file *file)
 959{
 960	int i;
 961	u32 val, bd_read_offset;
 962	struct flexrm_ring *ring;
 963
 964	seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
 965		   "Ring#", "BD_Read", "BD_Write",
 966		   "Cmpl_Read", "Submitted", "Completed");
 967
 968	for (i = 0; i < mbox->num_rings; i++) {
 969		ring = &mbox->rings[i];
 970		bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
 971		val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
 972		bd_read_offset *= RING_DESC_SIZE;
 973		bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
 974					ring->bd_dma_base);
 975		seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
 976			   ring->num,
 977			   (u32)bd_read_offset,
 978			   (u32)ring->bd_write_offset,
 979			   (u32)ring->cmpl_read_offset,
 980			   (u32)atomic_read(&ring->msg_send_count),
 981			   (u32)atomic_read(&ring->msg_cmpl_count));
 982	}
 983}
 984
 985static int flexrm_new_request(struct flexrm_ring *ring,
 986				struct brcm_message *batch_msg,
 987				struct brcm_message *msg)
 988{
 989	void *next;
 990	unsigned long flags;
 991	u32 val, count, nhcnt;
 992	u32 read_offset, write_offset;
 993	bool exit_cleanup = false;
 994	int ret = 0, reqid;
 995
 996	/* Do sanity check on message */
 997	if (!flexrm_sanity_check(msg))
 998		return -EIO;
 999	msg->error = 0;
1000
1001	/* If no requests possible then save data pointer and goto done. */
1002	spin_lock_irqsave(&ring->lock, flags);
1003	reqid = bitmap_find_free_region(ring->requests_bmap,
1004					RING_MAX_REQ_COUNT, 0);
1005	spin_unlock_irqrestore(&ring->lock, flags);
1006	if (reqid < 0)
1007		return -ENOSPC;
1008	ring->requests[reqid] = msg;
1009
1010	/* Do DMA mappings for the message */
1011	ret = flexrm_dma_map(ring->mbox->dev, msg);
1012	if (ret < 0) {
1013		ring->requests[reqid] = NULL;
1014		spin_lock_irqsave(&ring->lock, flags);
1015		bitmap_release_region(ring->requests_bmap, reqid, 0);
1016		spin_unlock_irqrestore(&ring->lock, flags);
1017		return ret;
1018	}
1019
1020	/* Determine current HW BD read offset */
1021	read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
1022	val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
1023	read_offset *= RING_DESC_SIZE;
1024	read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);
1025
1026	/*
1027	 * Number required descriptors = number of non-header descriptors +
1028	 *				 number of header descriptors +
1029	 *				 1x null descriptor
1030	 */
1031	nhcnt = flexrm_estimate_nonheader_desc_count(msg);
1032	count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;
1033
1034	/* Check for available descriptor space. */
1035	write_offset = ring->bd_write_offset;
1036	while (count) {
1037		if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))
1038			count--;
1039		write_offset += RING_DESC_SIZE;
1040		if (write_offset == RING_BD_SIZE)
1041			write_offset = 0x0;
1042		if (write_offset == read_offset)
1043			break;
1044	}
1045	if (count) {
1046		ret = -ENOSPC;
1047		exit_cleanup = true;
1048		goto exit;
1049	}
1050
1051	/* Write descriptors to ring */
1052	next = flexrm_write_descs(msg, nhcnt, reqid,
1053			ring->bd_base + ring->bd_write_offset,
1054			RING_BD_TOGGLE_VALID(ring->bd_write_offset),
1055			ring->bd_base, ring->bd_base + RING_BD_SIZE);
1056	if (IS_ERR(next)) {
1057		ret = PTR_ERR(next);
1058		exit_cleanup = true;
1059		goto exit;
1060	}
1061
1062	/* Save ring BD write offset */
1063	ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
1064
1065	/* Increment number of messages sent */
1066	atomic_inc_return(&ring->msg_send_count);
1067
1068exit:
1069	/* Update error status in message */
1070	msg->error = ret;
1071
1072	/* Cleanup if we failed */
1073	if (exit_cleanup) {
1074		flexrm_dma_unmap(ring->mbox->dev, msg);
1075		ring->requests[reqid] = NULL;
1076		spin_lock_irqsave(&ring->lock, flags);
1077		bitmap_release_region(ring->requests_bmap, reqid, 0);
1078		spin_unlock_irqrestore(&ring->lock, flags);
1079	}
1080
1081	return ret;
1082}
1083
1084static int flexrm_process_completions(struct flexrm_ring *ring)
1085{
1086	u64 desc;
1087	int err, count = 0;
1088	unsigned long flags;
1089	struct brcm_message *msg = NULL;
1090	u32 reqid, cmpl_read_offset, cmpl_write_offset;
1091	struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];
1092
1093	spin_lock_irqsave(&ring->lock, flags);
1094
1095	/*
1096	 * Get current completion read and write offset
1097	 *
1098	 * Note: We should read completion write pointer atleast once
1099	 * after we get a MSI interrupt because HW maintains internal
1100	 * MSI status which will allow next MSI interrupt only after
1101	 * completion write pointer is read.
1102	 */
1103	cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
1104	cmpl_write_offset *= RING_DESC_SIZE;
1105	cmpl_read_offset = ring->cmpl_read_offset;
1106	ring->cmpl_read_offset = cmpl_write_offset;
1107
1108	spin_unlock_irqrestore(&ring->lock, flags);
1109
1110	/* For each completed request notify mailbox clients */
1111	reqid = 0;
1112	while (cmpl_read_offset != cmpl_write_offset) {
1113		/* Dequeue next completion descriptor */
1114		desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));
1115
1116		/* Next read offset */
1117		cmpl_read_offset += RING_DESC_SIZE;
1118		if (cmpl_read_offset == RING_CMPL_SIZE)
1119			cmpl_read_offset = 0;
1120
1121		/* Decode error from completion descriptor */
1122		err = flexrm_cmpl_desc_to_error(desc);
1123		if (err < 0) {
1124			dev_warn(ring->mbox->dev,
1125			"ring%d got completion desc=0x%lx with error %d\n",
1126			ring->num, (unsigned long)desc, err);
1127		}
1128
1129		/* Determine request id from completion descriptor */
1130		reqid = flexrm_cmpl_desc_to_reqid(desc);
1131
1132		/* Determine message pointer based on reqid */
1133		msg = ring->requests[reqid];
1134		if (!msg) {
1135			dev_warn(ring->mbox->dev,
1136			"ring%d null msg pointer for completion desc=0x%lx\n",
1137			ring->num, (unsigned long)desc);
1138			continue;
1139		}
1140
1141		/* Release reqid for recycling */
1142		ring->requests[reqid] = NULL;
1143		spin_lock_irqsave(&ring->lock, flags);
1144		bitmap_release_region(ring->requests_bmap, reqid, 0);
1145		spin_unlock_irqrestore(&ring->lock, flags);
1146
1147		/* Unmap DMA mappings */
1148		flexrm_dma_unmap(ring->mbox->dev, msg);
1149
1150		/* Give-back message to mailbox client */
1151		msg->error = err;
1152		mbox_chan_received_data(chan, msg);
1153
1154		/* Increment number of completions processed */
1155		atomic_inc_return(&ring->msg_cmpl_count);
1156		count++;
1157	}
1158
1159	return count;
1160}
1161
1162/* ====== FlexRM Debugfs callbacks ====== */
1163
1164static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
1165{
1166	struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
1167
1168	/* Write config in file */
1169	flexrm_write_config_in_seqfile(mbox, file);
1170
1171	return 0;
1172}
1173
1174static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
1175{
1176	struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
1177
1178	/* Write stats in file */
1179	flexrm_write_stats_in_seqfile(mbox, file);
1180
1181	return 0;
1182}
1183
1184/* ====== FlexRM interrupt handler ===== */
1185
1186static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
1187{
1188	/* We only have MSI for completions so just wakeup IRQ thread */
1189	/* Ring related errors will be informed via completion descriptors */
1190
1191	return IRQ_WAKE_THREAD;
1192}
1193
1194static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)
1195{
1196	flexrm_process_completions(dev_id);
1197
1198	return IRQ_HANDLED;
1199}
1200
1201/* ====== FlexRM mailbox callbacks ===== */
1202
1203static int flexrm_send_data(struct mbox_chan *chan, void *data)
1204{
1205	int i, rc;
1206	struct flexrm_ring *ring = chan->con_priv;
1207	struct brcm_message *msg = data;
1208
1209	if (msg->type == BRCM_MESSAGE_BATCH) {
1210		for (i = msg->batch.msgs_queued;
1211		     i < msg->batch.msgs_count; i++) {
1212			rc = flexrm_new_request(ring, msg,
1213						 &msg->batch.msgs[i]);
1214			if (rc) {
1215				msg->error = rc;
1216				return rc;
1217			}
1218			msg->batch.msgs_queued++;
1219		}
1220		return 0;
1221	}
1222
1223	return flexrm_new_request(ring, NULL, data);
1224}
1225
1226static bool flexrm_peek_data(struct mbox_chan *chan)
1227{
1228	int cnt = flexrm_process_completions(chan->con_priv);
1229
1230	return (cnt > 0) ? true : false;
1231}
1232
1233static int flexrm_startup(struct mbox_chan *chan)
1234{
1235	u64 d;
1236	u32 val, off;
1237	int ret = 0;
1238	dma_addr_t next_addr;
1239	struct flexrm_ring *ring = chan->con_priv;
1240
1241	/* Allocate BD memory */
1242	ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,
1243				       GFP_KERNEL, &ring->bd_dma_base);
1244	if (!ring->bd_base) {
1245		dev_err(ring->mbox->dev,
1246			"can't allocate BD memory for ring%d\n",
1247			ring->num);
1248		ret = -ENOMEM;
1249		goto fail;
1250	}
1251
1252	/* Configure next table pointer entries in BD memory */
1253	for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {
1254		next_addr = off + RING_DESC_SIZE;
1255		if (next_addr == RING_BD_SIZE)
1256			next_addr = 0;
1257		next_addr += ring->bd_dma_base;
1258		if (RING_BD_ALIGN_CHECK(next_addr))
1259			d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),
1260						    next_addr);
1261		else
1262			d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));
1263		flexrm_write_desc(ring->bd_base + off, d);
1264	}
1265
1266	/* Allocate completion memory */
1267	ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool,
1268					 GFP_KERNEL, &ring->cmpl_dma_base);
1269	if (!ring->cmpl_base) {
1270		dev_err(ring->mbox->dev,
1271			"can't allocate completion memory for ring%d\n",
1272			ring->num);
1273		ret = -ENOMEM;
1274		goto fail_free_bd_memory;
1275	}
1276
1277	/* Request IRQ */
1278	if (ring->irq == UINT_MAX) {
1279		dev_err(ring->mbox->dev,
1280			"ring%d IRQ not available\n", ring->num);
1281		ret = -ENODEV;
1282		goto fail_free_cmpl_memory;
1283	}
1284	ret = request_threaded_irq(ring->irq,
1285				   flexrm_irq_event,
1286				   flexrm_irq_thread,
1287				   0, dev_name(ring->mbox->dev), ring);
1288	if (ret) {
1289		dev_err(ring->mbox->dev,
1290			"failed to request ring%d IRQ\n", ring->num);
1291		goto fail_free_cmpl_memory;
1292	}
1293	ring->irq_requested = true;
1294
1295	/* Set IRQ affinity hint */
1296	ring->irq_aff_hint = CPU_MASK_NONE;
1297	val = ring->mbox->num_rings;
1298	val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
1299	cpumask_set_cpu((ring->num / val) % num_online_cpus(),
1300			&ring->irq_aff_hint);
1301	ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint);
1302	if (ret) {
1303		dev_err(ring->mbox->dev,
1304			"failed to set IRQ affinity hint for ring%d\n",
1305			ring->num);
1306		goto fail_free_irq;
1307	}
1308
1309	/* Disable/inactivate ring */
1310	writel_relaxed(0x0, ring->regs + RING_CONTROL);
1311
1312	/* Program BD start address */
1313	val = BD_START_ADDR_VALUE(ring->bd_dma_base);
1314	writel_relaxed(val, ring->regs + RING_BD_START_ADDR);
1315
1316	/* BD write pointer will be same as HW write pointer */
1317	ring->bd_write_offset =
1318			readl_relaxed(ring->regs + RING_BD_WRITE_PTR);
1319	ring->bd_write_offset *= RING_DESC_SIZE;
1320
1321	/* Program completion start address */
1322	val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
1323	writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
1324
1325	/* Completion read pointer will be same as HW write pointer */
1326	ring->cmpl_read_offset =
1327			readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
1328	ring->cmpl_read_offset *= RING_DESC_SIZE;
1329
1330	/* Read ring Tx, Rx, and Outstanding counts to clear */
1331	readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);
1332	readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);
1333	readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);
1334	readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);
1335	readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);
1336
1337	/* Configure RING_MSI_CONTROL */
1338	val = 0;
1339	val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);
1340	val |= BIT(MSI_ENABLE_SHIFT);
1341	val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
1342	writel_relaxed(val, ring->regs + RING_MSI_CONTROL);
1343
1344	/* Enable/activate ring */
1345	val = BIT(CONTROL_ACTIVE_SHIFT);
1346	writel_relaxed(val, ring->regs + RING_CONTROL);
1347
1348	/* Reset stats to zero */
1349	atomic_set(&ring->msg_send_count, 0);
1350	atomic_set(&ring->msg_cmpl_count, 0);
1351
1352	return 0;
1353
1354fail_free_irq:
1355	free_irq(ring->irq, ring);
1356	ring->irq_requested = false;
1357fail_free_cmpl_memory:
1358	dma_pool_free(ring->mbox->cmpl_pool,
1359		      ring->cmpl_base, ring->cmpl_dma_base);
1360	ring->cmpl_base = NULL;
1361fail_free_bd_memory:
1362	dma_pool_free(ring->mbox->bd_pool,
1363		      ring->bd_base, ring->bd_dma_base);
1364	ring->bd_base = NULL;
1365fail:
1366	return ret;
1367}
1368
1369static void flexrm_shutdown(struct mbox_chan *chan)
1370{
1371	u32 reqid;
1372	unsigned int timeout;
1373	struct brcm_message *msg;
1374	struct flexrm_ring *ring = chan->con_priv;
1375
1376	/* Disable/inactivate ring */
1377	writel_relaxed(0x0, ring->regs + RING_CONTROL);
1378
1379	/* Set ring flush state */
1380	timeout = 1000; /* timeout of 1s */
1381	writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),
1382			ring->regs + RING_CONTROL);
1383	do {
1384		if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &
1385		    FLUSH_DONE_MASK)
1386			break;
1387		mdelay(1);
1388	} while (--timeout);
1389	if (!timeout)
1390		dev_err(ring->mbox->dev,
1391			"setting ring%d flush state timedout\n", ring->num);
1392
1393	/* Clear ring flush state */
1394	timeout = 1000; /* timeout of 1s */
1395	writel_relaxed(0x0, ring->regs + RING_CONTROL);
1396	do {
1397		if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) &
1398		      FLUSH_DONE_MASK))
1399			break;
1400		mdelay(1);
1401	} while (--timeout);
1402	if (!timeout)
1403		dev_err(ring->mbox->dev,
1404			"clearing ring%d flush state timedout\n", ring->num);
1405
1406	/* Abort all in-flight requests */
1407	for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {
1408		msg = ring->requests[reqid];
1409		if (!msg)
1410			continue;
1411
1412		/* Release reqid for recycling */
1413		ring->requests[reqid] = NULL;
1414
1415		/* Unmap DMA mappings */
1416		flexrm_dma_unmap(ring->mbox->dev, msg);
1417
1418		/* Give-back message to mailbox client */
1419		msg->error = -EIO;
1420		mbox_chan_received_data(chan, msg);
1421	}
1422
1423	/* Clear requests bitmap */
1424	bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
1425
1426	/* Release IRQ */
1427	if (ring->irq_requested) {
1428		irq_set_affinity_hint(ring->irq, NULL);
1429		free_irq(ring->irq, ring);
1430		ring->irq_requested = false;
1431	}
1432
1433	/* Free-up completion descriptor ring */
1434	if (ring->cmpl_base) {
1435		dma_pool_free(ring->mbox->cmpl_pool,
1436			      ring->cmpl_base, ring->cmpl_dma_base);
1437		ring->cmpl_base = NULL;
1438	}
1439
1440	/* Free-up BD descriptor ring */
1441	if (ring->bd_base) {
1442		dma_pool_free(ring->mbox->bd_pool,
1443			      ring->bd_base, ring->bd_dma_base);
1444		ring->bd_base = NULL;
1445	}
1446}
1447
1448static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
1449	.send_data	= flexrm_send_data,
1450	.startup	= flexrm_startup,
1451	.shutdown	= flexrm_shutdown,
1452	.peek_data	= flexrm_peek_data,
1453};
1454
1455static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,
1456					const struct of_phandle_args *pa)
1457{
1458	struct mbox_chan *chan;
1459	struct flexrm_ring *ring;
1460
1461	if (pa->args_count < 3)
1462		return ERR_PTR(-EINVAL);
1463
1464	if (pa->args[0] >= cntlr->num_chans)
1465		return ERR_PTR(-ENOENT);
1466
1467	if (pa->args[1] > MSI_COUNT_MASK)
1468		return ERR_PTR(-EINVAL);
1469
1470	if (pa->args[2] > MSI_TIMER_VAL_MASK)
1471		return ERR_PTR(-EINVAL);
1472
1473	chan = &cntlr->chans[pa->args[0]];
1474	ring = chan->con_priv;
1475	ring->msi_count_threshold = pa->args[1];
1476	ring->msi_timer_val = pa->args[2];
1477
1478	return chan;
1479}
1480
1481/* ====== FlexRM platform driver ===== */
1482
1483static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)
1484{
1485	struct device *dev = msi_desc_to_dev(desc);
1486	struct flexrm_mbox *mbox = dev_get_drvdata(dev);
1487	struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index];
1488
1489	/* Configure per-Ring MSI registers */
1490	writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);
1491	writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);
1492	writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);
1493}
1494
1495static int flexrm_mbox_probe(struct platform_device *pdev)
1496{
1497	int index, ret = 0;
1498	void __iomem *regs;
1499	void __iomem *regs_end;
1500	struct msi_desc *desc;
1501	struct resource *iomem;
1502	struct flexrm_ring *ring;
1503	struct flexrm_mbox *mbox;
1504	struct device *dev = &pdev->dev;
1505
1506	/* Allocate driver mailbox struct */
1507	mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
1508	if (!mbox) {
1509		ret = -ENOMEM;
1510		goto fail;
1511	}
1512	mbox->dev = dev;
1513	platform_set_drvdata(pdev, mbox);
1514
1515	/* Get resource for registers */
1516	iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1517	if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {
1518		ret = -ENODEV;
1519		goto fail;
1520	}
1521
1522	/* Map registers of all rings */
1523	mbox->regs = devm_ioremap_resource(&pdev->dev, iomem);
1524	if (IS_ERR(mbox->regs)) {
1525		ret = PTR_ERR(mbox->regs);
1526		dev_err(&pdev->dev, "Failed to remap mailbox regs: %d\n", ret);
1527		goto fail;
1528	}
1529	regs_end = mbox->regs + resource_size(iomem);
1530
1531	/* Scan and count available rings */
1532	mbox->num_rings = 0;
1533	for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {
1534		if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)
1535			mbox->num_rings++;
1536	}
1537	if (!mbox->num_rings) {
1538		ret = -ENODEV;
1539		goto fail;
1540	}
1541
1542	/* Allocate driver ring structs */
1543	ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);
1544	if (!ring) {
1545		ret = -ENOMEM;
1546		goto fail;
1547	}
1548	mbox->rings = ring;
1549
1550	/* Initialize members of driver ring structs */
1551	regs = mbox->regs;
1552	for (index = 0; index < mbox->num_rings; index++) {
1553		ring = &mbox->rings[index];
1554		ring->num = index;
1555		ring->mbox = mbox;
1556		while ((regs < regs_end) &&
1557		       (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))
1558			regs += RING_REGS_SIZE;
1559		if (regs_end <= regs) {
1560			ret = -ENODEV;
1561			goto fail;
1562		}
1563		ring->regs = regs;
1564		regs += RING_REGS_SIZE;
1565		ring->irq = UINT_MAX;
1566		ring->irq_requested = false;
1567		ring->msi_timer_val = MSI_TIMER_VAL_MASK;
1568		ring->msi_count_threshold = 0x1;
1569		memset(ring->requests, 0, sizeof(ring->requests));
1570		ring->bd_base = NULL;
1571		ring->bd_dma_base = 0;
1572		ring->cmpl_base = NULL;
1573		ring->cmpl_dma_base = 0;
1574		atomic_set(&ring->msg_send_count, 0);
1575		atomic_set(&ring->msg_cmpl_count, 0);
1576		spin_lock_init(&ring->lock);
1577		bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
1578		ring->cmpl_read_offset = 0;
1579	}
1580
1581	/* FlexRM is capable of 40-bit physical addresses only */
1582	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
1583	if (ret) {
1584		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1585		if (ret)
1586			goto fail;
1587	}
1588
1589	/* Create DMA pool for ring BD memory */
1590	mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,
1591					1 << RING_BD_ALIGN_ORDER, 0);
1592	if (!mbox->bd_pool) {
1593		ret = -ENOMEM;
1594		goto fail;
1595	}
1596
1597	/* Create DMA pool for ring completion memory */
1598	mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,
1599					  1 << RING_CMPL_ALIGN_ORDER, 0);
1600	if (!mbox->cmpl_pool) {
1601		ret = -ENOMEM;
1602		goto fail_destroy_bd_pool;
1603	}
1604
1605	/* Allocate platform MSIs for each ring */
1606	ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings,
1607						flexrm_mbox_msi_write);
1608	if (ret)
1609		goto fail_destroy_cmpl_pool;
1610
1611	/* Save alloced IRQ numbers for each ring */
1612	for_each_msi_entry(desc, dev) {
1613		ring = &mbox->rings[desc->platform.msi_index];
1614		ring->irq = desc->irq;
1615	}
1616
1617	/* Check availability of debugfs */
1618	if (!debugfs_initialized())
1619		goto skip_debugfs;
1620
1621	/* Create debugfs root entry */
1622	mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
1623
1624	/* Create debugfs config entry */
1625	debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root,
1626				    flexrm_debugfs_conf_show);
1627
1628	/* Create debugfs stats entry */
1629	debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root,
1630				    flexrm_debugfs_stats_show);
1631
1632skip_debugfs:
1633
1634	/* Initialize mailbox controller */
1635	mbox->controller.txdone_irq = false;
1636	mbox->controller.txdone_poll = false;
1637	mbox->controller.ops = &flexrm_mbox_chan_ops;
1638	mbox->controller.dev = dev;
1639	mbox->controller.num_chans = mbox->num_rings;
1640	mbox->controller.of_xlate = flexrm_mbox_of_xlate;
1641	mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,
1642				sizeof(*mbox->controller.chans), GFP_KERNEL);
1643	if (!mbox->controller.chans) {
1644		ret = -ENOMEM;
1645		goto fail_free_debugfs_root;
1646	}
1647	for (index = 0; index < mbox->num_rings; index++)
1648		mbox->controller.chans[index].con_priv = &mbox->rings[index];
1649
1650	/* Register mailbox controller */
1651	ret = devm_mbox_controller_register(dev, &mbox->controller);
1652	if (ret)
1653		goto fail_free_debugfs_root;
1654
1655	dev_info(dev, "registered flexrm mailbox with %d channels\n",
1656			mbox->controller.num_chans);
1657
1658	return 0;
1659
1660fail_free_debugfs_root:
1661	debugfs_remove_recursive(mbox->root);
1662	platform_msi_domain_free_irqs(dev);
1663fail_destroy_cmpl_pool:
1664	dma_pool_destroy(mbox->cmpl_pool);
1665fail_destroy_bd_pool:
1666	dma_pool_destroy(mbox->bd_pool);
1667fail:
1668	return ret;
1669}
1670
1671static int flexrm_mbox_remove(struct platform_device *pdev)
1672{
1673	struct device *dev = &pdev->dev;
1674	struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
1675
1676	debugfs_remove_recursive(mbox->root);
1677
1678	platform_msi_domain_free_irqs(dev);
1679
1680	dma_pool_destroy(mbox->cmpl_pool);
1681	dma_pool_destroy(mbox->bd_pool);
1682
1683	return 0;
1684}
1685
1686static const struct of_device_id flexrm_mbox_of_match[] = {
1687	{ .compatible = "brcm,iproc-flexrm-mbox", },
1688	{},
1689};
1690MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);
1691
1692static struct platform_driver flexrm_mbox_driver = {
1693	.driver = {
1694		.name = "brcm-flexrm-mbox",
1695		.of_match_table = flexrm_mbox_of_match,
1696	},
1697	.probe		= flexrm_mbox_probe,
1698	.remove		= flexrm_mbox_remove,
1699};
1700module_platform_driver(flexrm_mbox_driver);
1701
1702MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1703MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");
1704MODULE_LICENSE("GPL v2");