1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2015 Broadcom
   4 */
   5
   6/**
   7 * DOC: VC4 KMS
   8 *
   9 * This is the general code for implementing KMS mode setting that
  10 * doesn't clearly associate with any of the other objects (plane,
  11 * crtc, HDMI encoder).
  12 */
  13
  14#include <linux/clk.h>
  15
  16#include <drm/drm_atomic.h>
  17#include <drm/drm_atomic_helper.h>
  18#include <drm/drm_crtc.h>
  19#include <drm/drm_fourcc.h>
  20#include <drm/drm_gem_framebuffer_helper.h>
 
  21#include <drm/drm_probe_helper.h>
  22#include <drm/drm_vblank.h>
  23
  24#include "vc4_drv.h"
  25#include "vc4_regs.h"
  26
  27#define HVS_NUM_CHANNELS 3
  28
  29struct vc4_ctm_state {
  30	struct drm_private_state base;
  31	struct drm_color_ctm *ctm;
  32	int fifo;
  33};
  34
  35static struct vc4_ctm_state *
  36to_vc4_ctm_state(const struct drm_private_state *priv)
  37{
  38	return container_of(priv, struct vc4_ctm_state, base);
  39}
  40
  41struct vc4_hvs_state {
  42	struct drm_private_state base;
  43	unsigned long core_clock_rate;
  44
  45	struct {
  46		unsigned in_use: 1;
  47		unsigned long fifo_load;
  48		struct drm_crtc_commit *pending_commit;
  49	} fifo_state[HVS_NUM_CHANNELS];
  50};
  51
  52static struct vc4_hvs_state *
  53to_vc4_hvs_state(const struct drm_private_state *priv)
  54{
  55	return container_of(priv, struct vc4_hvs_state, base);
  56}
  57
  58struct vc4_load_tracker_state {
  59	struct drm_private_state base;
  60	u64 hvs_load;
  61	u64 membus_load;
  62};
  63
  64static struct vc4_load_tracker_state *
  65to_vc4_load_tracker_state(const struct drm_private_state *priv)
  66{
  67	return container_of(priv, struct vc4_load_tracker_state, base);
  68}
  69
  70static struct vc4_ctm_state *vc4_get_ctm_state(struct drm_atomic_state *state,
  71					       struct drm_private_obj *manager)
  72{
  73	struct drm_device *dev = state->dev;
  74	struct vc4_dev *vc4 = to_vc4_dev(dev);
  75	struct drm_private_state *priv_state;
  76	int ret;
  77
  78	ret = drm_modeset_lock(&vc4->ctm_state_lock, state->acquire_ctx);
  79	if (ret)
  80		return ERR_PTR(ret);
  81
  82	priv_state = drm_atomic_get_private_obj_state(state, manager);
  83	if (IS_ERR(priv_state))
  84		return ERR_CAST(priv_state);
  85
  86	return to_vc4_ctm_state(priv_state);
  87}
  88
  89static struct drm_private_state *
  90vc4_ctm_duplicate_state(struct drm_private_obj *obj)
  91{
  92	struct vc4_ctm_state *state;
  93
  94	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
  95	if (!state)
  96		return NULL;
  97
  98	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
  99
 100	return &state->base;
 101}
 102
 103static void vc4_ctm_destroy_state(struct drm_private_obj *obj,
 104				  struct drm_private_state *state)
 105{
 106	struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(state);
 107
 108	kfree(ctm_state);
 109}
 110
 111static const struct drm_private_state_funcs vc4_ctm_state_funcs = {
 112	.atomic_duplicate_state = vc4_ctm_duplicate_state,
 113	.atomic_destroy_state = vc4_ctm_destroy_state,
 114};
 115
 116static void vc4_ctm_obj_fini(struct drm_device *dev, void *unused)
 117{
 118	struct vc4_dev *vc4 = to_vc4_dev(dev);
 119
 120	drm_atomic_private_obj_fini(&vc4->ctm_manager);
 121}
 122
 123static int vc4_ctm_obj_init(struct vc4_dev *vc4)
 124{
 125	struct vc4_ctm_state *ctm_state;
 126
 127	drm_modeset_lock_init(&vc4->ctm_state_lock);
 128
 129	ctm_state = kzalloc(sizeof(*ctm_state), GFP_KERNEL);
 130	if (!ctm_state)
 131		return -ENOMEM;
 132
 133	drm_atomic_private_obj_init(&vc4->base, &vc4->ctm_manager, &ctm_state->base,
 134				    &vc4_ctm_state_funcs);
 135
 136	return drmm_add_action_or_reset(&vc4->base, vc4_ctm_obj_fini, NULL);
 137}
 138
 139/* Converts a DRM S31.32 value to the HW S0.9 format. */
 140static u16 vc4_ctm_s31_32_to_s0_9(u64 in)
 141{
 142	u16 r;
 143
 144	/* Sign bit. */
 145	r = in & BIT_ULL(63) ? BIT(9) : 0;
 146
 147	if ((in & GENMASK_ULL(62, 32)) > 0) {
 148		/* We have zero integer bits so we can only saturate here. */
 149		r |= GENMASK(8, 0);
 150	} else {
 151		/* Otherwise take the 9 most important fractional bits. */
 152		r |= (in >> 23) & GENMASK(8, 0);
 153	}
 154
 155	return r;
 156}
 157
 158static void
 159vc4_ctm_commit(struct vc4_dev *vc4, struct drm_atomic_state *state)
 160{
 161	struct vc4_hvs *hvs = vc4->hvs;
 162	struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(vc4->ctm_manager.state);
 163	struct drm_color_ctm *ctm = ctm_state->ctm;
 164
 165	if (ctm_state->fifo) {
 166		HVS_WRITE(SCALER_OLEDCOEF2,
 167			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[0]),
 168					SCALER_OLEDCOEF2_R_TO_R) |
 169			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[3]),
 170					SCALER_OLEDCOEF2_R_TO_G) |
 171			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[6]),
 172					SCALER_OLEDCOEF2_R_TO_B));
 173		HVS_WRITE(SCALER_OLEDCOEF1,
 174			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[1]),
 175					SCALER_OLEDCOEF1_G_TO_R) |
 176			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[4]),
 177					SCALER_OLEDCOEF1_G_TO_G) |
 178			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[7]),
 179					SCALER_OLEDCOEF1_G_TO_B));
 180		HVS_WRITE(SCALER_OLEDCOEF0,
 181			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[2]),
 182					SCALER_OLEDCOEF0_B_TO_R) |
 183			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[5]),
 184					SCALER_OLEDCOEF0_B_TO_G) |
 185			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[8]),
 186					SCALER_OLEDCOEF0_B_TO_B));
 187	}
 188
 189	HVS_WRITE(SCALER_OLEDOFFS,
 190		  VC4_SET_FIELD(ctm_state->fifo, SCALER_OLEDOFFS_DISPFIFO));
 191}
 192
 193static struct vc4_hvs_state *
 194vc4_hvs_get_new_global_state(struct drm_atomic_state *state)
 195{
 196	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 197	struct drm_private_state *priv_state;
 198
 199	priv_state = drm_atomic_get_new_private_obj_state(state, &vc4->hvs_channels);
 200	if (!priv_state)
 201		return ERR_PTR(-EINVAL);
 202
 203	return to_vc4_hvs_state(priv_state);
 204}
 205
 206static struct vc4_hvs_state *
 207vc4_hvs_get_old_global_state(struct drm_atomic_state *state)
 208{
 209	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 210	struct drm_private_state *priv_state;
 211
 212	priv_state = drm_atomic_get_old_private_obj_state(state, &vc4->hvs_channels);
 213	if (!priv_state)
 214		return ERR_PTR(-EINVAL);
 215
 216	return to_vc4_hvs_state(priv_state);
 217}
 218
 219static struct vc4_hvs_state *
 220vc4_hvs_get_global_state(struct drm_atomic_state *state)
 221{
 222	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 223	struct drm_private_state *priv_state;
 224
 225	priv_state = drm_atomic_get_private_obj_state(state, &vc4->hvs_channels);
 226	if (IS_ERR(priv_state))
 227		return ERR_CAST(priv_state);
 228
 229	return to_vc4_hvs_state(priv_state);
 230}
 231
 232static void vc4_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 233				     struct drm_atomic_state *state)
 234{
 235	struct vc4_hvs *hvs = vc4->hvs;
 236	struct drm_crtc_state *crtc_state;
 237	struct drm_crtc *crtc;
 238	unsigned int i;
 239
 240	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 241		struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 242		struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
 243		u32 dispctrl;
 244		u32 dsp3_mux;
 245
 246		if (!crtc_state->active)
 247			continue;
 248
 249		if (vc4_state->assigned_channel != 2)
 250			continue;
 251
 252		/*
 253		 * SCALER_DISPCTRL_DSP3 = X, where X < 2 means 'connect DSP3 to
 254		 * FIFO X'.
 255		 * SCALER_DISPCTRL_DSP3 = 3 means 'disable DSP 3'.
 256		 *
 257		 * DSP3 is connected to FIFO2 unless the transposer is
 258		 * enabled. In this case, FIFO 2 is directly accessed by the
 259		 * TXP IP, and we need to disable the FIFO2 -> pixelvalve1
 260		 * route.
 261		 */
 262		if (vc4_crtc->feeds_txp)
 263			dsp3_mux = VC4_SET_FIELD(3, SCALER_DISPCTRL_DSP3_MUX);
 264		else
 265			dsp3_mux = VC4_SET_FIELD(2, SCALER_DISPCTRL_DSP3_MUX);
 266
 267		dispctrl = HVS_READ(SCALER_DISPCTRL) &
 268			   ~SCALER_DISPCTRL_DSP3_MUX_MASK;
 269		HVS_WRITE(SCALER_DISPCTRL, dispctrl | dsp3_mux);
 270	}
 271}
 272
 273static void vc5_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 274				     struct drm_atomic_state *state)
 275{
 276	struct vc4_hvs *hvs = vc4->hvs;
 277	struct drm_crtc_state *crtc_state;
 278	struct drm_crtc *crtc;
 279	unsigned char mux;
 280	unsigned int i;
 281	u32 reg;
 282
 283	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 284		struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
 285		struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 286		unsigned int channel = vc4_state->assigned_channel;
 287
 288		if (!vc4_state->update_muxing)
 289			continue;
 290
 291		switch (vc4_crtc->data->hvs_output) {
 292		case 2:
 293			drm_WARN_ON(&vc4->base,
 294				    VC4_GET_FIELD(HVS_READ(SCALER_DISPCTRL),
 295						  SCALER_DISPCTRL_DSP3_MUX) == channel);
 296
 297			mux = (channel == 2) ? 0 : 1;
 298			reg = HVS_READ(SCALER_DISPECTRL);
 299			HVS_WRITE(SCALER_DISPECTRL,
 300				  (reg & ~SCALER_DISPECTRL_DSP2_MUX_MASK) |
 301				  VC4_SET_FIELD(mux, SCALER_DISPECTRL_DSP2_MUX));
 302			break;
 303
 304		case 3:
 305			if (channel == VC4_HVS_CHANNEL_DISABLED)
 306				mux = 3;
 307			else
 308				mux = channel;
 309
 310			reg = HVS_READ(SCALER_DISPCTRL);
 311			HVS_WRITE(SCALER_DISPCTRL,
 312				  (reg & ~SCALER_DISPCTRL_DSP3_MUX_MASK) |
 313				  VC4_SET_FIELD(mux, SCALER_DISPCTRL_DSP3_MUX));
 314			break;
 315
 316		case 4:
 317			if (channel == VC4_HVS_CHANNEL_DISABLED)
 318				mux = 3;
 319			else
 320				mux = channel;
 321
 322			reg = HVS_READ(SCALER_DISPEOLN);
 323			HVS_WRITE(SCALER_DISPEOLN,
 324				  (reg & ~SCALER_DISPEOLN_DSP4_MUX_MASK) |
 325				  VC4_SET_FIELD(mux, SCALER_DISPEOLN_DSP4_MUX));
 326
 327			break;
 328
 329		case 5:
 330			if (channel == VC4_HVS_CHANNEL_DISABLED)
 331				mux = 3;
 332			else
 333				mux = channel;
 334
 335			reg = HVS_READ(SCALER_DISPDITHER);
 336			HVS_WRITE(SCALER_DISPDITHER,
 337				  (reg & ~SCALER_DISPDITHER_DSP5_MUX_MASK) |
 338				  VC4_SET_FIELD(mux, SCALER_DISPDITHER_DSP5_MUX));
 339			break;
 340
 341		default:
 342			break;
 343		}
 344	}
 345}
 346
 347static void vc4_atomic_commit_tail(struct drm_atomic_state *state)
 348{
 349	struct drm_device *dev = state->dev;
 350	struct vc4_dev *vc4 = to_vc4_dev(dev);
 351	struct vc4_hvs *hvs = vc4->hvs;
 
 352	struct drm_crtc_state *new_crtc_state;
 353	struct vc4_hvs_state *new_hvs_state;
 354	struct drm_crtc *crtc;
 355	struct vc4_hvs_state *old_hvs_state;
 356	unsigned int channel;
 357	int i;
 358
 359	old_hvs_state = vc4_hvs_get_old_global_state(state);
 360	if (WARN_ON(IS_ERR(old_hvs_state)))
 361		return;
 362
 363	new_hvs_state = vc4_hvs_get_new_global_state(state);
 364	if (WARN_ON(IS_ERR(new_hvs_state)))
 365		return;
 366
 367	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
 368		struct vc4_crtc_state *vc4_crtc_state;
 369
 370		if (!new_crtc_state->commit)
 371			continue;
 372
 373		vc4_crtc_state = to_vc4_crtc_state(new_crtc_state);
 374		vc4_hvs_mask_underrun(hvs, vc4_crtc_state->assigned_channel);
 375	}
 376
 377	for (channel = 0; channel < HVS_NUM_CHANNELS; channel++) {
 378		struct drm_crtc_commit *commit;
 
 
 
 
 
 
 
 
 
 379		int ret;
 380
 381		if (!old_hvs_state->fifo_state[channel].in_use)
 382			continue;
 383
 384		commit = old_hvs_state->fifo_state[channel].pending_commit;
 385		if (!commit)
 386			continue;
 387
 388		ret = drm_crtc_commit_wait(commit);
 389		if (ret)
 390			drm_err(dev, "Timed out waiting for commit\n");
 391
 392		drm_crtc_commit_put(commit);
 393		old_hvs_state->fifo_state[channel].pending_commit = NULL;
 394	}
 395
 396	if (vc4->is_vc5) {
 397		unsigned long state_rate = max(old_hvs_state->core_clock_rate,
 398					       new_hvs_state->core_clock_rate);
 399		unsigned long core_rate = clamp_t(unsigned long, state_rate,
 400						  500000000, hvs->max_core_rate);
 401
 402		drm_dbg(dev, "Raising the core clock at %lu Hz\n", core_rate);
 403
 404		/*
 405		 * Do a temporary request on the core clock during the
 406		 * modeset.
 407		 */
 408		WARN_ON(clk_set_min_rate(hvs->core_clk, core_rate));
 409	}
 410
 411	drm_atomic_helper_commit_modeset_disables(dev, state);
 412
 413	vc4_ctm_commit(vc4, state);
 414
 415	if (vc4->is_vc5)
 416		vc5_hvs_pv_muxing_commit(vc4, state);
 417	else
 418		vc4_hvs_pv_muxing_commit(vc4, state);
 419
 420	drm_atomic_helper_commit_planes(dev, state,
 421					DRM_PLANE_COMMIT_ACTIVE_ONLY);
 422
 423	drm_atomic_helper_commit_modeset_enables(dev, state);
 424
 425	drm_atomic_helper_fake_vblank(state);
 426
 427	drm_atomic_helper_commit_hw_done(state);
 428
 429	drm_atomic_helper_wait_for_flip_done(dev, state);
 430
 431	drm_atomic_helper_cleanup_planes(dev, state);
 432
 433	if (vc4->is_vc5) {
 434		unsigned long core_rate = min_t(unsigned long,
 435						hvs->max_core_rate,
 436						new_hvs_state->core_clock_rate);
 437
 438		drm_dbg(dev, "Running the core clock at %lu Hz\n", core_rate);
 439
 440		/*
 441		 * Request a clock rate based on the current HVS
 442		 * requirements.
 443		 */
 444		WARN_ON(clk_set_min_rate(hvs->core_clk, core_rate));
 445
 446		drm_dbg(dev, "Core clock actual rate: %lu Hz\n",
 447			clk_get_rate(hvs->core_clk));
 448	}
 449}
 450
 451static int vc4_atomic_commit_setup(struct drm_atomic_state *state)
 452{
 453	struct drm_crtc_state *crtc_state;
 454	struct vc4_hvs_state *hvs_state;
 455	struct drm_crtc *crtc;
 456	unsigned int i;
 457
 458	hvs_state = vc4_hvs_get_new_global_state(state);
 459	if (WARN_ON(IS_ERR(hvs_state)))
 460		return PTR_ERR(hvs_state);
 461
 462	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 463		struct vc4_crtc_state *vc4_crtc_state =
 464			to_vc4_crtc_state(crtc_state);
 465		unsigned int channel =
 466			vc4_crtc_state->assigned_channel;
 467
 468		if (channel == VC4_HVS_CHANNEL_DISABLED)
 469			continue;
 470
 471		if (!hvs_state->fifo_state[channel].in_use)
 472			continue;
 473
 474		hvs_state->fifo_state[channel].pending_commit =
 475			drm_crtc_commit_get(crtc_state->commit);
 476	}
 477
 478	return 0;
 479}
 480
 481static struct drm_framebuffer *vc4_fb_create(struct drm_device *dev,
 482					     struct drm_file *file_priv,
 483					     const struct drm_mode_fb_cmd2 *mode_cmd)
 484{
 485	struct vc4_dev *vc4 = to_vc4_dev(dev);
 486	struct drm_mode_fb_cmd2 mode_cmd_local;
 487
 488	if (WARN_ON_ONCE(vc4->is_vc5))
 489		return ERR_PTR(-ENODEV);
 490
 491	/* If the user didn't specify a modifier, use the
 492	 * vc4_set_tiling_ioctl() state for the BO.
 493	 */
 494	if (!(mode_cmd->flags & DRM_MODE_FB_MODIFIERS)) {
 495		struct drm_gem_object *gem_obj;
 496		struct vc4_bo *bo;
 497
 498		gem_obj = drm_gem_object_lookup(file_priv,
 499						mode_cmd->handles[0]);
 500		if (!gem_obj) {
 501			DRM_DEBUG("Failed to look up GEM BO %d\n",
 502				  mode_cmd->handles[0]);
 503			return ERR_PTR(-ENOENT);
 504		}
 505		bo = to_vc4_bo(gem_obj);
 506
 507		mode_cmd_local = *mode_cmd;
 508
 509		if (bo->t_format) {
 510			mode_cmd_local.modifier[0] =
 511				DRM_FORMAT_MOD_BROADCOM_VC4_T_TILED;
 512		} else {
 513			mode_cmd_local.modifier[0] = DRM_FORMAT_MOD_NONE;
 514		}
 515
 516		drm_gem_object_put(gem_obj);
 517
 518		mode_cmd = &mode_cmd_local;
 519	}
 520
 521	return drm_gem_fb_create(dev, file_priv, mode_cmd);
 522}
 523
 524/* Our CTM has some peculiar limitations: we can only enable it for one CRTC
 525 * at a time and the HW only supports S0.9 scalars. To account for the latter,
 526 * we don't allow userland to set a CTM that we have no hope of approximating.
 527 */
 528static int
 529vc4_ctm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 530{
 531	struct vc4_dev *vc4 = to_vc4_dev(dev);
 532	struct vc4_ctm_state *ctm_state = NULL;
 533	struct drm_crtc *crtc;
 534	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 535	struct drm_color_ctm *ctm;
 536	int i;
 537
 538	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 539		/* CTM is being disabled. */
 540		if (!new_crtc_state->ctm && old_crtc_state->ctm) {
 541			ctm_state = vc4_get_ctm_state(state, &vc4->ctm_manager);
 542			if (IS_ERR(ctm_state))
 543				return PTR_ERR(ctm_state);
 544			ctm_state->fifo = 0;
 545		}
 546	}
 547
 548	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 549		if (new_crtc_state->ctm == old_crtc_state->ctm)
 550			continue;
 551
 552		if (!ctm_state) {
 553			ctm_state = vc4_get_ctm_state(state, &vc4->ctm_manager);
 554			if (IS_ERR(ctm_state))
 555				return PTR_ERR(ctm_state);
 556		}
 557
 558		/* CTM is being enabled or the matrix changed. */
 559		if (new_crtc_state->ctm) {
 560			struct vc4_crtc_state *vc4_crtc_state =
 561				to_vc4_crtc_state(new_crtc_state);
 562
 563			/* fifo is 1-based since 0 disables CTM. */
 564			int fifo = vc4_crtc_state->assigned_channel + 1;
 565
 566			/* Check userland isn't trying to turn on CTM for more
 567			 * than one CRTC at a time.
 568			 */
 569			if (ctm_state->fifo && ctm_state->fifo != fifo) {
 570				DRM_DEBUG_DRIVER("Too many CTM configured\n");
 571				return -EINVAL;
 572			}
 573
 574			/* Check we can approximate the specified CTM.
 575			 * We disallow scalars |c| > 1.0 since the HW has
 576			 * no integer bits.
 577			 */
 578			ctm = new_crtc_state->ctm->data;
 579			for (i = 0; i < ARRAY_SIZE(ctm->matrix); i++) {
 580				u64 val = ctm->matrix[i];
 581
 582				val &= ~BIT_ULL(63);
 583				if (val > BIT_ULL(32))
 584					return -EINVAL;
 585			}
 586
 587			ctm_state->fifo = fifo;
 588			ctm_state->ctm = ctm;
 589		}
 590	}
 591
 592	return 0;
 593}
 594
 595static int vc4_load_tracker_atomic_check(struct drm_atomic_state *state)
 596{
 597	struct drm_plane_state *old_plane_state, *new_plane_state;
 598	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 599	struct vc4_load_tracker_state *load_state;
 600	struct drm_private_state *priv_state;
 601	struct drm_plane *plane;
 602	int i;
 603
 
 
 
 604	priv_state = drm_atomic_get_private_obj_state(state,
 605						      &vc4->load_tracker);
 606	if (IS_ERR(priv_state))
 607		return PTR_ERR(priv_state);
 608
 609	load_state = to_vc4_load_tracker_state(priv_state);
 610	for_each_oldnew_plane_in_state(state, plane, old_plane_state,
 611				       new_plane_state, i) {
 612		struct vc4_plane_state *vc4_plane_state;
 613
 614		if (old_plane_state->fb && old_plane_state->crtc) {
 615			vc4_plane_state = to_vc4_plane_state(old_plane_state);
 616			load_state->membus_load -= vc4_plane_state->membus_load;
 617			load_state->hvs_load -= vc4_plane_state->hvs_load;
 618		}
 619
 620		if (new_plane_state->fb && new_plane_state->crtc) {
 621			vc4_plane_state = to_vc4_plane_state(new_plane_state);
 622			load_state->membus_load += vc4_plane_state->membus_load;
 623			load_state->hvs_load += vc4_plane_state->hvs_load;
 624		}
 625	}
 626
 627	/* Don't check the load when the tracker is disabled. */
 628	if (!vc4->load_tracker_enabled)
 629		return 0;
 630
 631	/* The absolute limit is 2Gbyte/sec, but let's take a margin to let
 632	 * the system work when other blocks are accessing the memory.
 633	 */
 634	if (load_state->membus_load > SZ_1G + SZ_512M)
 635		return -ENOSPC;
 636
 637	/* HVS clock is supposed to run @ 250Mhz, let's take a margin and
 638	 * consider the maximum number of cycles is 240M.
 639	 */
 640	if (load_state->hvs_load > 240000000ULL)
 641		return -ENOSPC;
 642
 643	return 0;
 644}
 645
 646static struct drm_private_state *
 647vc4_load_tracker_duplicate_state(struct drm_private_obj *obj)
 648{
 649	struct vc4_load_tracker_state *state;
 650
 651	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
 652	if (!state)
 653		return NULL;
 654
 655	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 656
 657	return &state->base;
 658}
 659
 660static void vc4_load_tracker_destroy_state(struct drm_private_obj *obj,
 661					   struct drm_private_state *state)
 662{
 663	struct vc4_load_tracker_state *load_state;
 664
 665	load_state = to_vc4_load_tracker_state(state);
 666	kfree(load_state);
 667}
 668
 669static const struct drm_private_state_funcs vc4_load_tracker_state_funcs = {
 670	.atomic_duplicate_state = vc4_load_tracker_duplicate_state,
 671	.atomic_destroy_state = vc4_load_tracker_destroy_state,
 672};
 673
 674static void vc4_load_tracker_obj_fini(struct drm_device *dev, void *unused)
 675{
 676	struct vc4_dev *vc4 = to_vc4_dev(dev);
 677
 
 
 
 678	drm_atomic_private_obj_fini(&vc4->load_tracker);
 679}
 680
 681static int vc4_load_tracker_obj_init(struct vc4_dev *vc4)
 682{
 683	struct vc4_load_tracker_state *load_state;
 684
 
 
 
 685	load_state = kzalloc(sizeof(*load_state), GFP_KERNEL);
 686	if (!load_state)
 687		return -ENOMEM;
 688
 689	drm_atomic_private_obj_init(&vc4->base, &vc4->load_tracker,
 690				    &load_state->base,
 691				    &vc4_load_tracker_state_funcs);
 692
 693	return drmm_add_action_or_reset(&vc4->base, vc4_load_tracker_obj_fini, NULL);
 694}
 695
 696static struct drm_private_state *
 697vc4_hvs_channels_duplicate_state(struct drm_private_obj *obj)
 698{
 699	struct vc4_hvs_state *old_state = to_vc4_hvs_state(obj->state);
 700	struct vc4_hvs_state *state;
 701	unsigned int i;
 702
 703	state = kzalloc(sizeof(*state), GFP_KERNEL);
 704	if (!state)
 705		return NULL;
 706
 707	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 708
 
 709	for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 710		state->fifo_state[i].in_use = old_state->fifo_state[i].in_use;
 711		state->fifo_state[i].fifo_load = old_state->fifo_state[i].fifo_load;
 712	}
 713
 714	state->core_clock_rate = old_state->core_clock_rate;
 
 
 
 
 
 715
 716	return &state->base;
 717}
 718
 719static void vc4_hvs_channels_destroy_state(struct drm_private_obj *obj,
 720					   struct drm_private_state *state)
 721{
 722	struct vc4_hvs_state *hvs_state = to_vc4_hvs_state(state);
 723	unsigned int i;
 724
 725	for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 726		if (!hvs_state->fifo_state[i].pending_commit)
 727			continue;
 728
 729		drm_crtc_commit_put(hvs_state->fifo_state[i].pending_commit);
 730	}
 731
 732	kfree(hvs_state);
 733}
 734
 735static void vc4_hvs_channels_print_state(struct drm_printer *p,
 736					 const struct drm_private_state *state)
 737{
 738	struct vc4_hvs_state *hvs_state = to_vc4_hvs_state(state);
 739	unsigned int i;
 740
 741	drm_printf(p, "HVS State\n");
 742	drm_printf(p, "\tCore Clock Rate: %lu\n", hvs_state->core_clock_rate);
 743
 744	for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 745		drm_printf(p, "\tChannel %d\n", i);
 746		drm_printf(p, "\t\tin use=%d\n", hvs_state->fifo_state[i].in_use);
 747		drm_printf(p, "\t\tload=%lu\n", hvs_state->fifo_state[i].fifo_load);
 748	}
 749}
 750
 751static const struct drm_private_state_funcs vc4_hvs_state_funcs = {
 752	.atomic_duplicate_state = vc4_hvs_channels_duplicate_state,
 753	.atomic_destroy_state = vc4_hvs_channels_destroy_state,
 754	.atomic_print_state = vc4_hvs_channels_print_state,
 755};
 756
 757static void vc4_hvs_channels_obj_fini(struct drm_device *dev, void *unused)
 758{
 759	struct vc4_dev *vc4 = to_vc4_dev(dev);
 760
 761	drm_atomic_private_obj_fini(&vc4->hvs_channels);
 762}
 763
 764static int vc4_hvs_channels_obj_init(struct vc4_dev *vc4)
 765{
 766	struct vc4_hvs_state *state;
 767
 768	state = kzalloc(sizeof(*state), GFP_KERNEL);
 769	if (!state)
 770		return -ENOMEM;
 771
 772	drm_atomic_private_obj_init(&vc4->base, &vc4->hvs_channels,
 773				    &state->base,
 774				    &vc4_hvs_state_funcs);
 775
 776	return drmm_add_action_or_reset(&vc4->base, vc4_hvs_channels_obj_fini, NULL);
 777}
 778
 779/*
 780 * The BCM2711 HVS has up to 7 outputs connected to the pixelvalves and
 781 * the TXP (and therefore all the CRTCs found on that platform).
 782 *
 783 * The naive (and our initial) implementation would just iterate over
 784 * all the active CRTCs, try to find a suitable FIFO, and then remove it
 785 * from the pool of available FIFOs. However, there are a few corner
 786 * cases that need to be considered:
 787 *
 788 * - When running in a dual-display setup (so with two CRTCs involved),
 789 *   we can update the state of a single CRTC (for example by changing
 790 *   its mode using xrandr under X11) without affecting the other. In
 791 *   this case, the other CRTC wouldn't be in the state at all, so we
 792 *   need to consider all the running CRTCs in the DRM device to assign
 793 *   a FIFO, not just the one in the state.
 794 *
 795 * - To fix the above, we can't use drm_atomic_get_crtc_state on all
 796 *   enabled CRTCs to pull their CRTC state into the global state, since
 797 *   a page flip would start considering their vblank to complete. Since
 798 *   we don't have a guarantee that they are actually active, that
 799 *   vblank might never happen, and shouldn't even be considered if we
 800 *   want to do a page flip on a single CRTC. That can be tested by
 801 *   doing a modetest -v first on HDMI1 and then on HDMI0.
 802 *
 803 * - Since we need the pixelvalve to be disabled and enabled back when
 804 *   the FIFO is changed, we should keep the FIFO assigned for as long
 805 *   as the CRTC is enabled, only considering it free again once that
 806 *   CRTC has been disabled. This can be tested by booting X11 on a
 807 *   single display, and changing the resolution down and then back up.
 808 */
 809static int vc4_pv_muxing_atomic_check(struct drm_device *dev,
 810				      struct drm_atomic_state *state)
 811{
 812	struct vc4_hvs_state *hvs_new_state;
 813	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 814	struct drm_crtc *crtc;
 815	unsigned int unassigned_channels = 0;
 816	unsigned int i;
 817
 818	hvs_new_state = vc4_hvs_get_global_state(state);
 819	if (IS_ERR(hvs_new_state))
 820		return PTR_ERR(hvs_new_state);
 821
 822	for (i = 0; i < ARRAY_SIZE(hvs_new_state->fifo_state); i++)
 823		if (!hvs_new_state->fifo_state[i].in_use)
 824			unassigned_channels |= BIT(i);
 825
 826	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 827		struct vc4_crtc_state *old_vc4_crtc_state =
 828			to_vc4_crtc_state(old_crtc_state);
 829		struct vc4_crtc_state *new_vc4_crtc_state =
 830			to_vc4_crtc_state(new_crtc_state);
 831		struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 832		unsigned int matching_channels;
 833		unsigned int channel;
 834
 835		drm_dbg(dev, "%s: Trying to find a channel.\n", crtc->name);
 836
 837		/* Nothing to do here, let's skip it */
 838		if (old_crtc_state->enable == new_crtc_state->enable) {
 839			if (new_crtc_state->enable)
 840				drm_dbg(dev, "%s: Already enabled, reusing channel %d.\n",
 841					crtc->name, new_vc4_crtc_state->assigned_channel);
 842			else
 843				drm_dbg(dev, "%s: Disabled, ignoring.\n", crtc->name);
 844
 845			continue;
 846		}
 847
 848		/* Muxing will need to be modified, mark it as such */
 849		new_vc4_crtc_state->update_muxing = true;
 850
 851		/* If we're disabling our CRTC, we put back our channel */
 852		if (!new_crtc_state->enable) {
 853			channel = old_vc4_crtc_state->assigned_channel;
 854
 855			drm_dbg(dev, "%s: Disabling, Freeing channel %d\n",
 856				crtc->name, channel);
 857
 858			hvs_new_state->fifo_state[channel].in_use = false;
 859			new_vc4_crtc_state->assigned_channel = VC4_HVS_CHANNEL_DISABLED;
 860			continue;
 861		}
 862
 863		/*
 864		 * The problem we have to solve here is that we have
 865		 * up to 7 encoders, connected to up to 6 CRTCs.
 866		 *
 867		 * Those CRTCs, depending on the instance, can be
 868		 * routed to 1, 2 or 3 HVS FIFOs, and we need to set
 869		 * the change the muxing between FIFOs and outputs in
 870		 * the HVS accordingly.
 871		 *
 872		 * It would be pretty hard to come up with an
 873		 * algorithm that would generically solve
 874		 * this. However, the current routing trees we support
 875		 * allow us to simplify a bit the problem.
 876		 *
 877		 * Indeed, with the current supported layouts, if we
 878		 * try to assign in the ascending crtc index order the
 879		 * FIFOs, we can't fall into the situation where an
 880		 * earlier CRTC that had multiple routes is assigned
 881		 * one that was the only option for a later CRTC.
 882		 *
 883		 * If the layout changes and doesn't give us that in
 884		 * the future, we will need to have something smarter,
 885		 * but it works so far.
 886		 */
 887		matching_channels = unassigned_channels & vc4_crtc->data->hvs_available_channels;
 888		if (!matching_channels)
 889			return -EINVAL;
 890
 891		channel = ffs(matching_channels) - 1;
 892
 893		drm_dbg(dev, "Assigned HVS channel %d to CRTC %s\n", channel, crtc->name);
 894		new_vc4_crtc_state->assigned_channel = channel;
 895		unassigned_channels &= ~BIT(channel);
 896		hvs_new_state->fifo_state[channel].in_use = true;
 897	}
 898
 899	return 0;
 900}
 901
 902static int
 903vc4_core_clock_atomic_check(struct drm_atomic_state *state)
 904{
 905	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 906	struct drm_private_state *priv_state;
 907	struct vc4_hvs_state *hvs_new_state;
 908	struct vc4_load_tracker_state *load_state;
 909	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 910	struct drm_crtc *crtc;
 911	unsigned int num_outputs;
 912	unsigned long pixel_rate;
 913	unsigned long cob_rate;
 914	unsigned int i;
 915
 916	priv_state = drm_atomic_get_private_obj_state(state,
 917						      &vc4->load_tracker);
 918	if (IS_ERR(priv_state))
 919		return PTR_ERR(priv_state);
 920
 921	load_state = to_vc4_load_tracker_state(priv_state);
 922
 923	hvs_new_state = vc4_hvs_get_global_state(state);
 924	if (IS_ERR(hvs_new_state))
 925		return PTR_ERR(hvs_new_state);
 926
 927	for_each_oldnew_crtc_in_state(state, crtc,
 928				      old_crtc_state,
 929				      new_crtc_state,
 930				      i) {
 931		if (old_crtc_state->active) {
 932			struct vc4_crtc_state *old_vc4_state =
 933				to_vc4_crtc_state(old_crtc_state);
 934			unsigned int channel = old_vc4_state->assigned_channel;
 935
 936			hvs_new_state->fifo_state[channel].fifo_load = 0;
 937		}
 938
 939		if (new_crtc_state->active) {
 940			struct vc4_crtc_state *new_vc4_state =
 941				to_vc4_crtc_state(new_crtc_state);
 942			unsigned int channel = new_vc4_state->assigned_channel;
 943
 944			hvs_new_state->fifo_state[channel].fifo_load =
 945				new_vc4_state->hvs_load;
 946		}
 947	}
 948
 949	cob_rate = 0;
 950	num_outputs = 0;
 951	for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 952		if (!hvs_new_state->fifo_state[i].in_use)
 953			continue;
 954
 955		num_outputs++;
 956		cob_rate = max_t(unsigned long,
 957				 hvs_new_state->fifo_state[i].fifo_load,
 958				 cob_rate);
 959	}
 960
 961	pixel_rate = load_state->hvs_load;
 962	if (num_outputs > 1) {
 963		pixel_rate = (pixel_rate * 40) / 100;
 964	} else {
 965		pixel_rate = (pixel_rate * 60) / 100;
 966	}
 967
 968	hvs_new_state->core_clock_rate = max(cob_rate, pixel_rate);
 969
 970	return 0;
 971}
 972
 973
 974static int
 975vc4_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 976{
 977	int ret;
 978
 979	ret = vc4_pv_muxing_atomic_check(dev, state);
 980	if (ret)
 981		return ret;
 982
 983	ret = vc4_ctm_atomic_check(dev, state);
 984	if (ret < 0)
 985		return ret;
 986
 987	ret = drm_atomic_helper_check(dev, state);
 988	if (ret)
 989		return ret;
 990
 991	ret = vc4_load_tracker_atomic_check(state);
 992	if (ret)
 993		return ret;
 994
 995	return vc4_core_clock_atomic_check(state);
 996}
 997
 998static struct drm_mode_config_helper_funcs vc4_mode_config_helpers = {
 999	.atomic_commit_setup	= vc4_atomic_commit_setup,
1000	.atomic_commit_tail	= vc4_atomic_commit_tail,
1001};
1002
1003static const struct drm_mode_config_funcs vc4_mode_funcs = {
1004	.atomic_check = vc4_atomic_check,
1005	.atomic_commit = drm_atomic_helper_commit,
1006	.fb_create = vc4_fb_create,
1007};
1008
1009static const struct drm_mode_config_funcs vc5_mode_funcs = {
1010	.atomic_check = vc4_atomic_check,
1011	.atomic_commit = drm_atomic_helper_commit,
1012	.fb_create = drm_gem_fb_create,
1013};
1014
1015int vc4_kms_load(struct drm_device *dev)
1016{
1017	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
 
1018	int ret;
1019
1020	/*
1021	 * The limits enforced by the load tracker aren't relevant for
1022	 * the BCM2711, but the load tracker computations are used for
1023	 * the core clock rate calculation.
1024	 */
1025	if (!vc4->is_vc5) {
1026		/* Start with the load tracker enabled. Can be
1027		 * disabled through the debugfs load_tracker file.
1028		 */
1029		vc4->load_tracker_enabled = true;
1030	}
1031
1032	/* Set support for vblank irq fast disable, before drm_vblank_init() */
1033	dev->vblank_disable_immediate = true;
1034
 
1035	ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
1036	if (ret < 0) {
1037		dev_err(dev->dev, "failed to initialize vblank\n");
1038		return ret;
1039	}
1040
1041	if (vc4->is_vc5) {
1042		dev->mode_config.max_width = 7680;
1043		dev->mode_config.max_height = 7680;
1044	} else {
1045		dev->mode_config.max_width = 2048;
1046		dev->mode_config.max_height = 2048;
1047	}
1048
1049	dev->mode_config.funcs = vc4->is_vc5 ? &vc5_mode_funcs : &vc4_mode_funcs;
1050	dev->mode_config.helper_private = &vc4_mode_config_helpers;
1051	dev->mode_config.preferred_depth = 24;
1052	dev->mode_config.async_page_flip = true;
1053
1054	ret = vc4_ctm_obj_init(vc4);
1055	if (ret)
1056		return ret;
1057
1058	ret = vc4_load_tracker_obj_init(vc4);
1059	if (ret)
1060		return ret;
1061
1062	ret = vc4_hvs_channels_obj_init(vc4);
1063	if (ret)
1064		return ret;
1065
1066	drm_mode_config_reset(dev);
1067
1068	drm_kms_helper_poll_init(dev);
1069
1070	return 0;
1071}