1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
   4 */
   5
   6#include <linux/clk.h>
   7#include <linux/delay.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/export.h>
  10#include <linux/interrupt.h>
  11#include <linux/kernel.h>
  12#include <linux/module.h>
  13#include <linux/of.h>
  14#include <linux/of_platform.h>
  15#include <linux/platform_device.h>
  16#include <linux/slab.h>
  17#include <linux/sort.h>
  18#include <linux/tegra-icc.h>
  19
  20#include <soc/tegra/fuse.h>
  21
  22#include "mc.h"
  23
  24static const struct of_device_id tegra_mc_of_match[] = {
  25#ifdef CONFIG_ARCH_TEGRA_2x_SOC
  26	{ .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
  27#endif
  28#ifdef CONFIG_ARCH_TEGRA_3x_SOC
  29	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
  30#endif
  31#ifdef CONFIG_ARCH_TEGRA_114_SOC
  32	{ .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
  33#endif
  34#ifdef CONFIG_ARCH_TEGRA_124_SOC
  35	{ .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
  36#endif
  37#ifdef CONFIG_ARCH_TEGRA_132_SOC
  38	{ .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
  39#endif
  40#ifdef CONFIG_ARCH_TEGRA_210_SOC
  41	{ .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
  42#endif
  43#ifdef CONFIG_ARCH_TEGRA_186_SOC
  44	{ .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
  45#endif
  46#ifdef CONFIG_ARCH_TEGRA_194_SOC
  47	{ .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
  48#endif
  49#ifdef CONFIG_ARCH_TEGRA_234_SOC
  50	{ .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
  51#endif
  52	{ /* sentinel */ }
  53};
  54MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
  55
  56static void tegra_mc_devm_action_put_device(void *data)
  57{
  58	struct tegra_mc *mc = data;
  59
  60	put_device(mc->dev);
  61}
  62
  63/**
  64 * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
  65 * @dev: device pointer for the consumer device
  66 *
  67 * This function will search for the Memory Controller node in a device-tree
  68 * and retrieve the Memory Controller handle.
  69 *
  70 * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
  71 */
  72struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
  73{
  74	struct platform_device *pdev;
  75	struct device_node *np;
  76	struct tegra_mc *mc;
  77	int err;
  78
  79	np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
  80	if (!np)
  81		return ERR_PTR(-ENOENT);
  82
  83	pdev = of_find_device_by_node(np);
  84	of_node_put(np);
  85	if (!pdev)
  86		return ERR_PTR(-ENODEV);
  87
  88	mc = platform_get_drvdata(pdev);
  89	if (!mc) {
  90		put_device(&pdev->dev);
  91		return ERR_PTR(-EPROBE_DEFER);
  92	}
  93
  94	err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc);
  95	if (err)
  96		return ERR_PTR(err);
  97
  98	return mc;
  99}
 100EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
 101
 102int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev)
 103{
 104	if (mc->soc->ops && mc->soc->ops->probe_device)
 105		return mc->soc->ops->probe_device(mc, dev);
 106
 107	return 0;
 108}
 109EXPORT_SYMBOL_GPL(tegra_mc_probe_device);
 110
 111int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id,
 112                               phys_addr_t *base, u64 *size)
 113{
 114	u32 offset;
 115
 116	if (id < 1 || id >= mc->soc->num_carveouts)
 117		return -EINVAL;
 118
 119	if (id < 6)
 120		offset = 0xc0c + 0x50 * (id - 1);
 121	else
 122		offset = 0x2004 + 0x50 * (id - 6);
 123
 124	*base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x0);
 125#ifdef CONFIG_PHYS_ADDR_T_64BIT
 126	*base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x4) << 32;
 127#endif
 128
 129	if (size)
 130		*size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x8) << 17;
 131
 132	return 0;
 133}
 134EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info);
 135
 136static int tegra_mc_block_dma_common(struct tegra_mc *mc,
 137				     const struct tegra_mc_reset *rst)
 138{
 139	unsigned long flags;
 140	u32 value;
 141
 142	spin_lock_irqsave(&mc->lock, flags);
 143
 144	value = mc_readl(mc, rst->control) | BIT(rst->bit);
 145	mc_writel(mc, value, rst->control);
 146
 147	spin_unlock_irqrestore(&mc->lock, flags);
 148
 149	return 0;
 150}
 151
 152static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
 153				       const struct tegra_mc_reset *rst)
 154{
 155	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
 156}
 157
 158static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
 159				       const struct tegra_mc_reset *rst)
 160{
 161	unsigned long flags;
 162	u32 value;
 163
 164	spin_lock_irqsave(&mc->lock, flags);
 165
 166	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
 167	mc_writel(mc, value, rst->control);
 168
 169	spin_unlock_irqrestore(&mc->lock, flags);
 170
 171	return 0;
 172}
 173
 174static int tegra_mc_reset_status_common(struct tegra_mc *mc,
 175					const struct tegra_mc_reset *rst)
 176{
 177	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
 178}
 179
 180const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
 181	.block_dma = tegra_mc_block_dma_common,
 182	.dma_idling = tegra_mc_dma_idling_common,
 183	.unblock_dma = tegra_mc_unblock_dma_common,
 184	.reset_status = tegra_mc_reset_status_common,
 185};
 186
 187static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
 188{
 189	return container_of(rcdev, struct tegra_mc, reset);
 190}
 191
 192static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
 193							unsigned long id)
 194{
 195	unsigned int i;
 196
 197	for (i = 0; i < mc->soc->num_resets; i++)
 198		if (mc->soc->resets[i].id == id)
 199			return &mc->soc->resets[i];
 200
 201	return NULL;
 202}
 203
 204static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
 205				    unsigned long id)
 206{
 207	struct tegra_mc *mc = reset_to_mc(rcdev);
 208	const struct tegra_mc_reset_ops *rst_ops;
 209	const struct tegra_mc_reset *rst;
 210	int retries = 500;
 211	int err;
 212
 213	rst = tegra_mc_reset_find(mc, id);
 214	if (!rst)
 215		return -ENODEV;
 216
 217	rst_ops = mc->soc->reset_ops;
 218	if (!rst_ops)
 219		return -ENODEV;
 220
 221	/* DMA flushing will fail if reset is already asserted */
 222	if (rst_ops->reset_status) {
 223		/* check whether reset is asserted */
 224		if (rst_ops->reset_status(mc, rst))
 225			return 0;
 226	}
 227
 228	if (rst_ops->block_dma) {
 229		/* block clients DMA requests */
 230		err = rst_ops->block_dma(mc, rst);
 231		if (err) {
 232			dev_err(mc->dev, "failed to block %s DMA: %d\n",
 233				rst->name, err);
 234			return err;
 235		}
 236	}
 237
 238	if (rst_ops->dma_idling) {
 239		/* wait for completion of the outstanding DMA requests */
 240		while (!rst_ops->dma_idling(mc, rst)) {
 241			if (!retries--) {
 242				dev_err(mc->dev, "failed to flush %s DMA\n",
 243					rst->name);
 244				return -EBUSY;
 245			}
 246
 247			usleep_range(10, 100);
 248		}
 249	}
 250
 251	if (rst_ops->hotreset_assert) {
 252		/* clear clients DMA requests sitting before arbitration */
 253		err = rst_ops->hotreset_assert(mc, rst);
 254		if (err) {
 255			dev_err(mc->dev, "failed to hot reset %s: %d\n",
 256				rst->name, err);
 257			return err;
 258		}
 259	}
 260
 261	return 0;
 262}
 263
 264static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
 265				      unsigned long id)
 266{
 267	struct tegra_mc *mc = reset_to_mc(rcdev);
 268	const struct tegra_mc_reset_ops *rst_ops;
 269	const struct tegra_mc_reset *rst;
 270	int err;
 271
 272	rst = tegra_mc_reset_find(mc, id);
 273	if (!rst)
 274		return -ENODEV;
 275
 276	rst_ops = mc->soc->reset_ops;
 277	if (!rst_ops)
 278		return -ENODEV;
 279
 280	if (rst_ops->hotreset_deassert) {
 281		/* take out client from hot reset */
 282		err = rst_ops->hotreset_deassert(mc, rst);
 283		if (err) {
 284			dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
 285				rst->name, err);
 286			return err;
 287		}
 288	}
 289
 290	if (rst_ops->unblock_dma) {
 291		/* allow new DMA requests to proceed to arbitration */
 292		err = rst_ops->unblock_dma(mc, rst);
 293		if (err) {
 294			dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
 295				rst->name, err);
 296			return err;
 297		}
 298	}
 299
 300	return 0;
 301}
 302
 303static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
 304				    unsigned long id)
 305{
 306	struct tegra_mc *mc = reset_to_mc(rcdev);
 307	const struct tegra_mc_reset_ops *rst_ops;
 308	const struct tegra_mc_reset *rst;
 309
 310	rst = tegra_mc_reset_find(mc, id);
 311	if (!rst)
 312		return -ENODEV;
 313
 314	rst_ops = mc->soc->reset_ops;
 315	if (!rst_ops)
 316		return -ENODEV;
 317
 318	return rst_ops->reset_status(mc, rst);
 319}
 320
 321static const struct reset_control_ops tegra_mc_reset_ops = {
 322	.assert = tegra_mc_hotreset_assert,
 323	.deassert = tegra_mc_hotreset_deassert,
 324	.status = tegra_mc_hotreset_status,
 325};
 326
 327static int tegra_mc_reset_setup(struct tegra_mc *mc)
 328{
 329	int err;
 330
 331	mc->reset.ops = &tegra_mc_reset_ops;
 332	mc->reset.owner = THIS_MODULE;
 333	mc->reset.of_node = mc->dev->of_node;
 334	mc->reset.of_reset_n_cells = 1;
 335	mc->reset.nr_resets = mc->soc->num_resets;
 336
 337	err = reset_controller_register(&mc->reset);
 338	if (err < 0)
 339		return err;
 340
 341	return 0;
 342}
 343
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 344int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
 345{
 346	unsigned int i;
 347	struct tegra_mc_timing *timing = NULL;
 348
 349	for (i = 0; i < mc->num_timings; i++) {
 350		if (mc->timings[i].rate == rate) {
 351			timing = &mc->timings[i];
 352			break;
 353		}
 354	}
 355
 356	if (!timing) {
 357		dev_err(mc->dev, "no memory timing registered for rate %lu\n",
 358			rate);
 359		return -EINVAL;
 360	}
 361
 362	for (i = 0; i < mc->soc->num_emem_regs; ++i)
 363		mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
 364
 365	return 0;
 366}
 367EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
 368
 369unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
 370{
 371	u8 dram_count;
 372
 373	dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
 374	dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
 375	dram_count++;
 376
 377	return dram_count;
 378}
 379EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
 380
 381#if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
 382    defined(CONFIG_ARCH_TEGRA_114_SOC) || \
 383    defined(CONFIG_ARCH_TEGRA_124_SOC) || \
 384    defined(CONFIG_ARCH_TEGRA_132_SOC) || \
 385    defined(CONFIG_ARCH_TEGRA_210_SOC)
 386static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 387{
 388	unsigned long long tick;
 389	unsigned int i;
 390	u32 value;
 391
 392	/* compute the number of MC clock cycles per tick */
 393	tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
 394	do_div(tick, NSEC_PER_SEC);
 395
 396	value = mc_readl(mc, MC_EMEM_ARB_CFG);
 397	value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
 398	value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
 399	mc_writel(mc, value, MC_EMEM_ARB_CFG);
 400
 401	/* write latency allowance defaults */
 402	for (i = 0; i < mc->soc->num_clients; i++) {
 403		const struct tegra_mc_client *client = &mc->soc->clients[i];
 404		u32 value;
 405
 406		value = mc_readl(mc, client->regs.la.reg);
 407		value &= ~(client->regs.la.mask << client->regs.la.shift);
 408		value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;
 409		mc_writel(mc, value, client->regs.la.reg);
 410	}
 411
 412	/* latch new values */
 413	mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
 414
 415	return 0;
 416}
 417
 418static int load_one_timing(struct tegra_mc *mc,
 419			   struct tegra_mc_timing *timing,
 420			   struct device_node *node)
 421{
 422	int err;
 423	u32 tmp;
 424
 425	err = of_property_read_u32(node, "clock-frequency", &tmp);
 426	if (err) {
 427		dev_err(mc->dev,
 428			"timing %pOFn: failed to read rate\n", node);
 429		return err;
 430	}
 431
 432	timing->rate = tmp;
 433	timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
 434					 sizeof(u32), GFP_KERNEL);
 435	if (!timing->emem_data)
 436		return -ENOMEM;
 437
 438	err = of_property_read_u32_array(node, "nvidia,emem-configuration",
 439					 timing->emem_data,
 440					 mc->soc->num_emem_regs);
 441	if (err) {
 442		dev_err(mc->dev,
 443			"timing %pOFn: failed to read EMEM configuration\n",
 444			node);
 445		return err;
 446	}
 447
 448	return 0;
 449}
 450
 451static int load_timings(struct tegra_mc *mc, struct device_node *node)
 452{
 
 453	struct tegra_mc_timing *timing;
 454	int child_count = of_get_child_count(node);
 455	int i = 0, err;
 456
 457	mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
 458				   GFP_KERNEL);
 459	if (!mc->timings)
 460		return -ENOMEM;
 461
 462	mc->num_timings = child_count;
 463
 464	for_each_child_of_node_scoped(node, child) {
 465		timing = &mc->timings[i++];
 466
 467		err = load_one_timing(mc, timing, child);
 468		if (err)
 
 469			return err;
 
 470	}
 471
 472	return 0;
 473}
 474
 475static int tegra_mc_setup_timings(struct tegra_mc *mc)
 476{
 
 477	u32 ram_code, node_ram_code;
 478	int err;
 479
 480	ram_code = tegra_read_ram_code();
 481
 482	mc->num_timings = 0;
 483
 484	for_each_child_of_node_scoped(mc->dev->of_node, node) {
 485		err = of_property_read_u32(node, "nvidia,ram-code",
 486					   &node_ram_code);
 487		if (err || (node_ram_code != ram_code))
 488			continue;
 489
 490		err = load_timings(mc, node);
 
 491		if (err)
 492			return err;
 493		break;
 494	}
 495
 496	if (mc->num_timings == 0)
 497		dev_warn(mc->dev,
 498			 "no memory timings for RAM code %u registered\n",
 499			 ram_code);
 500
 501	return 0;
 502}
 503
 504int tegra30_mc_probe(struct tegra_mc *mc)
 505{
 506	int err;
 507
 508	mc->clk = devm_clk_get_optional(mc->dev, "mc");
 509	if (IS_ERR(mc->clk)) {
 510		dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
 511		return PTR_ERR(mc->clk);
 512	}
 513
 514	/* ensure that debug features are disabled */
 515	mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
 516
 517	err = tegra_mc_setup_latency_allowance(mc);
 518	if (err < 0) {
 519		dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
 520		return err;
 521	}
 522
 523	err = tegra_mc_setup_timings(mc);
 524	if (err < 0) {
 525		dev_err(mc->dev, "failed to setup timings: %d\n", err);
 526		return err;
 527	}
 528
 529	return 0;
 530}
 531
 532const struct tegra_mc_ops tegra30_mc_ops = {
 533	.probe = tegra30_mc_probe,
 534	.handle_irq = tegra30_mc_handle_irq,
 535};
 536#endif
 537
 538static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status,
 539					  unsigned int *mc_channel)
 540{
 541	if ((status & mc->soc->ch_intmask) == 0)
 542		return -EINVAL;
 543
 544	*mc_channel = __ffs((status & mc->soc->ch_intmask) >>
 545			    mc->soc->global_intstatus_channel_shift);
 546
 547	return 0;
 548}
 549
 550static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc,
 551					  unsigned int channel)
 552{
 553	return BIT(channel) << mc->soc->global_intstatus_channel_shift;
 554}
 
 555
 556irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 557{
 558	struct tegra_mc *mc = data;
 559	unsigned int bit, channel;
 560	unsigned long status;
 
 561
 562	if (mc->soc->num_channels) {
 563		u32 global_status;
 564		int err;
 565
 566		global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS);
 567		err = mc_global_intstatus_to_channel(mc, global_status, &channel);
 568		if (err < 0) {
 569			dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n",
 570					    global_status);
 571			return IRQ_NONE;
 572		}
 573
 574		/* mask all interrupts to avoid flooding */
 575		status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;
 576	} else {
 577		status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 578	}
 579
 580	if (!status)
 581		return IRQ_NONE;
 582
 583	for_each_set_bit(bit, &status, 32) {
 584		const char *error = tegra_mc_status_names[bit] ?: "unknown";
 585		const char *client = "unknown", *desc;
 586		const char *direction, *secure;
 587		u32 status_reg, addr_reg;
 588		u32 intmask = BIT(bit);
 589		phys_addr_t addr = 0;
 590#ifdef CONFIG_PHYS_ADDR_T_64BIT
 591		u32 addr_hi_reg = 0;
 592#endif
 593		unsigned int i;
 594		char perm[7];
 595		u8 id, type;
 596		u32 value;
 597
 598		switch (intmask) {
 599		case MC_INT_DECERR_VPR:
 600			status_reg = MC_ERR_VPR_STATUS;
 601			addr_reg = MC_ERR_VPR_ADR;
 602			break;
 603
 604		case MC_INT_SECERR_SEC:
 605			status_reg = MC_ERR_SEC_STATUS;
 606			addr_reg = MC_ERR_SEC_ADR;
 607			break;
 608
 609		case MC_INT_DECERR_MTS:
 610			status_reg = MC_ERR_MTS_STATUS;
 611			addr_reg = MC_ERR_MTS_ADR;
 612			break;
 613
 614		case MC_INT_DECERR_GENERALIZED_CARVEOUT:
 615			status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS;
 616			addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR;
 617			break;
 618
 619		case MC_INT_DECERR_ROUTE_SANITY:
 620			status_reg = MC_ERR_ROUTE_SANITY_STATUS;
 621			addr_reg = MC_ERR_ROUTE_SANITY_ADR;
 622			break;
 623
 624		default:
 625			status_reg = MC_ERR_STATUS;
 626			addr_reg = MC_ERR_ADR;
 627
 628#ifdef CONFIG_PHYS_ADDR_T_64BIT
 629			if (mc->soc->has_addr_hi_reg)
 630				addr_hi_reg = MC_ERR_ADR_HI;
 631#endif
 632			break;
 633		}
 634
 635		if (mc->soc->num_channels)
 636			value = mc_ch_readl(mc, channel, status_reg);
 637		else
 638			value = mc_readl(mc, status_reg);
 639
 640#ifdef CONFIG_PHYS_ADDR_T_64BIT
 641		if (mc->soc->num_address_bits > 32) {
 642			if (addr_hi_reg) {
 643				if (mc->soc->num_channels)
 644					addr = mc_ch_readl(mc, channel, addr_hi_reg);
 645				else
 646					addr = mc_readl(mc, addr_hi_reg);
 647			} else {
 648				addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
 649					MC_ERR_STATUS_ADR_HI_MASK);
 650			}
 651			addr <<= 32;
 652		}
 653#endif
 654
 655		if (value & MC_ERR_STATUS_RW)
 656			direction = "write";
 657		else
 658			direction = "read";
 659
 660		if (value & MC_ERR_STATUS_SECURITY)
 661			secure = "secure ";
 662		else
 663			secure = "";
 664
 665		id = value & mc->soc->client_id_mask;
 666
 667		for (i = 0; i < mc->soc->num_clients; i++) {
 668			if (mc->soc->clients[i].id == id) {
 669				client = mc->soc->clients[i].name;
 670				break;
 671			}
 672		}
 673
 674		type = (value & MC_ERR_STATUS_TYPE_MASK) >>
 675		       MC_ERR_STATUS_TYPE_SHIFT;
 676		desc = tegra_mc_error_names[type];
 677
 678		switch (value & MC_ERR_STATUS_TYPE_MASK) {
 679		case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
 680			perm[0] = ' ';
 681			perm[1] = '[';
 682
 683			if (value & MC_ERR_STATUS_READABLE)
 684				perm[2] = 'R';
 685			else
 686				perm[2] = '-';
 687
 688			if (value & MC_ERR_STATUS_WRITABLE)
 689				perm[3] = 'W';
 690			else
 691				perm[3] = '-';
 692
 693			if (value & MC_ERR_STATUS_NONSECURE)
 694				perm[4] = '-';
 695			else
 696				perm[4] = 'S';
 697
 698			perm[5] = ']';
 699			perm[6] = '\0';
 700			break;
 701
 702		default:
 703			perm[0] = '\0';
 704			break;
 705		}
 706
 707		if (mc->soc->num_channels)
 708			value = mc_ch_readl(mc, channel, addr_reg);
 709		else
 710			value = mc_readl(mc, addr_reg);
 711		addr |= value;
 712
 713		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
 714				    client, secure, direction, &addr, error,
 715				    desc, perm);
 716	}
 717
 718	/* clear interrupts */
 719	if (mc->soc->num_channels) {
 720		mc_ch_writel(mc, channel, status, MC_INTSTATUS);
 721		mc_ch_writel(mc, MC_BROADCAST_CHANNEL,
 722			     mc_channel_to_global_intstatus(mc, channel),
 723			     MC_GLOBAL_INTSTATUS);
 724	} else {
 725		mc_writel(mc, status, MC_INTSTATUS);
 726	}
 727
 728	return IRQ_HANDLED;
 729}
 730
 731const char *const tegra_mc_status_names[32] = {
 732	[ 1] = "External interrupt",
 733	[ 6] = "EMEM address decode error",
 734	[ 7] = "GART page fault",
 735	[ 8] = "Security violation",
 736	[ 9] = "EMEM arbitration error",
 737	[10] = "Page fault",
 738	[11] = "Invalid APB ASID update",
 739	[12] = "VPR violation",
 740	[13] = "Secure carveout violation",
 741	[16] = "MTS carveout violation",
 742	[17] = "Generalized carveout violation",
 743	[20] = "Route Sanity error",
 744};
 745
 746const char *const tegra_mc_error_names[8] = {
 747	[2] = "EMEM decode error",
 748	[3] = "TrustZone violation",
 749	[4] = "Carveout violation",
 750	[6] = "SMMU translation error",
 751};
 752
 753struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, void *data)
 754{
 755	struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
 756	struct icc_node *node;
 
 
 
 757
 758	list_for_each_entry(node, &mc->provider.nodes, node_list) {
 759		if (node->id == spec->args[0])
 760			return node;
 761	}
 762
 763	/*
 764	 * If a client driver calls devm_of_icc_get() before the MC driver
 765	 * is probed, then return EPROBE_DEFER to the client driver.
 766	 */
 767	return ERR_PTR(-EPROBE_DEFER);
 768}
 769
 770static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak)
 771{
 772	*average = 0;
 773	*peak = 0;
 774
 775	return 0;
 776}
 
 777
 778static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst)
 779{
 780	return 0;
 781}
 782
 783const struct tegra_mc_icc_ops tegra_mc_icc_ops = {
 784	.xlate = tegra_mc_icc_xlate,
 785	.aggregate = icc_std_aggregate,
 786	.get_bw = tegra_mc_icc_get,
 787	.set = tegra_mc_icc_set,
 788};
 789
 790/*
 791 * Memory Controller (MC) has few Memory Clients that are issuing memory
 792 * bandwidth allocation requests to the MC interconnect provider. The MC
 793 * provider aggregates the requests and then sends the aggregated request
 794 * up to the External Memory Controller (EMC) interconnect provider which
 795 * re-configures hardware interface to External Memory (EMEM) in accordance
 796 * to the required bandwidth. Each MC interconnect node represents an
 797 * individual Memory Client.
 798 *
 799 * Memory interconnect topology:
 800 *
 801 *               +----+
 802 * +--------+    |    |
 803 * | TEXSRD +--->+    |
 804 * +--------+    |    |
 805 *               |    |    +-----+    +------+
 806 *    ...        | MC +--->+ EMC +--->+ EMEM |
 807 *               |    |    +-----+    +------+
 808 * +--------+    |    |
 809 * | DISP.. +--->+    |
 810 * +--------+    |    |
 811 *               +----+
 812 */
 813static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
 814{
 815	struct icc_node *node;
 816	unsigned int i;
 817	int err;
 818
 819	/* older device-trees don't have interconnect properties */
 820	if (!device_property_present(mc->dev, "#interconnect-cells") ||
 821	    !mc->soc->icc_ops)
 822		return 0;
 823
 824	mc->provider.dev = mc->dev;
 825	mc->provider.data = &mc->provider;
 826	mc->provider.set = mc->soc->icc_ops->set;
 827	mc->provider.aggregate = mc->soc->icc_ops->aggregate;
 828	mc->provider.get_bw = mc->soc->icc_ops->get_bw;
 829	mc->provider.xlate = mc->soc->icc_ops->xlate;
 830	mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
 831
 832	icc_provider_init(&mc->provider);
 833
 834	/* create Memory Controller node */
 835	node = icc_node_create(TEGRA_ICC_MC);
 836	if (IS_ERR(node))
 837		return PTR_ERR(node);
 838
 839	node->name = "Memory Controller";
 840	icc_node_add(node, &mc->provider);
 841
 842	/* link Memory Controller to External Memory Controller */
 843	err = icc_link_create(node, TEGRA_ICC_EMC);
 844	if (err)
 845		goto remove_nodes;
 846
 847	for (i = 0; i < mc->soc->num_clients; i++) {
 848		/* create MC client node */
 849		node = icc_node_create(mc->soc->clients[i].id);
 850		if (IS_ERR(node)) {
 851			err = PTR_ERR(node);
 852			goto remove_nodes;
 853		}
 854
 855		node->name = mc->soc->clients[i].name;
 856		icc_node_add(node, &mc->provider);
 857
 858		/* link Memory Client to Memory Controller */
 859		err = icc_link_create(node, TEGRA_ICC_MC);
 860		if (err)
 861			goto remove_nodes;
 862
 863		node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]);
 
 
 864	}
 865
 866	err = icc_provider_register(&mc->provider);
 867	if (err)
 868		goto remove_nodes;
 869
 870	return 0;
 871
 872remove_nodes:
 873	icc_nodes_remove(&mc->provider);
 874
 875	return err;
 876}
 877
 878static void tegra_mc_num_channel_enabled(struct tegra_mc *mc)
 879{
 880	unsigned int i;
 881	u32 value;
 882
 883	value = mc_ch_readl(mc, 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE);
 884	if (value <= 0) {
 885		mc->num_channels = mc->soc->num_channels;
 886		return;
 887	}
 888
 889	for (i = 0; i < 32; i++) {
 890		if (value & BIT(i))
 891			mc->num_channels++;
 892	}
 893}
 894
 895static int tegra_mc_probe(struct platform_device *pdev)
 896{
 
 897	struct tegra_mc *mc;
 
 898	u64 mask;
 899	int err;
 900
 901	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
 902	if (!mc)
 903		return -ENOMEM;
 904
 905	platform_set_drvdata(pdev, mc);
 906	spin_lock_init(&mc->lock);
 907	mc->soc = of_device_get_match_data(&pdev->dev);
 908	mc->dev = &pdev->dev;
 909
 910	mask = DMA_BIT_MASK(mc->soc->num_address_bits);
 911
 912	err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
 913	if (err < 0) {
 914		dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
 915		return err;
 916	}
 917
 918	/* length of MC tick in nanoseconds */
 919	mc->tick = 30;
 920
 921	mc->regs = devm_platform_ioremap_resource(pdev, 0);
 
 922	if (IS_ERR(mc->regs))
 923		return PTR_ERR(mc->regs);
 924
 925	mc->debugfs.root = debugfs_create_dir("mc", NULL);
 926
 927	if (mc->soc->ops && mc->soc->ops->probe) {
 928		err = mc->soc->ops->probe(mc);
 929		if (err < 0)
 930			return err;
 931	}
 932
 933	tegra_mc_num_channel_enabled(mc);
 
 
 
 
 
 
 
 934
 935	if (mc->soc->ops && mc->soc->ops->handle_irq) {
 936		mc->irq = platform_get_irq(pdev, 0);
 937		if (mc->irq < 0)
 938			return mc->irq;
 939
 940		WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
 941
 942		if (mc->soc->num_channels)
 943			mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,
 944				     MC_INTMASK);
 945		else
 946			mc_writel(mc, mc->soc->intmask, MC_INTMASK);
 947
 948		err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
 949				       dev_name(&pdev->dev), mc);
 
 950		if (err < 0) {
 951			dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
 952				err);
 953			return err;
 954		}
 955	}
 956
 957	if (mc->soc->reset_ops) {
 958		err = tegra_mc_reset_setup(mc);
 959		if (err < 0)
 960			dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);
 961	}
 962
 963	err = tegra_mc_interconnect_setup(mc);
 
 
 
 
 
 
 
 
 
 
 
 
 964	if (err < 0)
 965		dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
 966			err);
 967
 968	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
 969		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
 970		if (IS_ERR(mc->smmu)) {
 971			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
 972				PTR_ERR(mc->smmu));
 973			mc->smmu = NULL;
 974		}
 975	}
 976
 
 
 
 
 
 
 
 
 
 977	return 0;
 978}
 979
 980static void tegra_mc_sync_state(struct device *dev)
 981{
 982	struct tegra_mc *mc = dev_get_drvdata(dev);
 
 983
 984	/* check whether ICC provider is registered */
 985	if (mc->provider.dev == dev)
 986		icc_sync_state(dev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 987}
 988
 
 
 
 
 
 989static struct platform_driver tegra_mc_driver = {
 990	.driver = {
 991		.name = "tegra-mc",
 992		.of_match_table = tegra_mc_of_match,
 
 993		.suppress_bind_attrs = true,
 994		.sync_state = tegra_mc_sync_state,
 995	},
 996	.prevent_deferred_probe = true,
 997	.probe = tegra_mc_probe,
 998};
 999
1000static int tegra_mc_init(void)
1001{
1002	return platform_driver_register(&tegra_mc_driver);
1003}
1004arch_initcall(tegra_mc_init);
1005
1006MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
1007MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");