1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2015 Broadcom
   4 */
   5
   6/**
   7 * DOC: VC4 KMS
   8 *
   9 * This is the general code for implementing KMS mode setting that
  10 * doesn't clearly associate with any of the other objects (plane,
  11 * crtc, HDMI encoder).
  12 */
  13
  14#include <linux/clk.h>
  15#include <linux/sort.h>
  16
  17#include <drm/drm_atomic.h>
  18#include <drm/drm_atomic_helper.h>
  19#include <drm/drm_crtc.h>
  20#include <drm/drm_fourcc.h>
  21#include <drm/drm_gem_framebuffer_helper.h>
 
  22#include <drm/drm_probe_helper.h>
  23#include <drm/drm_vblank.h>
  24
  25#include "vc4_drv.h"
  26#include "vc4_regs.h"
  27
  28struct vc4_ctm_state {
  29	struct drm_private_state base;
  30	struct drm_color_ctm *ctm;
  31	int fifo;
  32};
  33
  34#define to_vc4_ctm_state(_state)				\
  35	container_of_const(_state, struct vc4_ctm_state, base)
 
 
  36
  37struct vc4_load_tracker_state {
  38	struct drm_private_state base;
  39	u64 hvs_load;
  40	u64 membus_load;
  41};
  42
  43#define to_vc4_load_tracker_state(_state)				\
  44	container_of_const(_state, struct vc4_load_tracker_state, base)
 
 
 
  45
  46static struct vc4_ctm_state *vc4_get_ctm_state(struct drm_atomic_state *state,
  47					       struct drm_private_obj *manager)
  48{
  49	struct drm_device *dev = state->dev;
  50	struct vc4_dev *vc4 = to_vc4_dev(dev);
  51	struct drm_private_state *priv_state;
  52	int ret;
  53
  54	ret = drm_modeset_lock(&vc4->ctm_state_lock, state->acquire_ctx);
  55	if (ret)
  56		return ERR_PTR(ret);
  57
  58	priv_state = drm_atomic_get_private_obj_state(state, manager);
  59	if (IS_ERR(priv_state))
  60		return ERR_CAST(priv_state);
  61
  62	return to_vc4_ctm_state(priv_state);
  63}
  64
  65static struct drm_private_state *
  66vc4_ctm_duplicate_state(struct drm_private_obj *obj)
  67{
  68	struct vc4_ctm_state *state;
  69
  70	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
  71	if (!state)
  72		return NULL;
  73
  74	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
  75
  76	return &state->base;
  77}
  78
  79static void vc4_ctm_destroy_state(struct drm_private_obj *obj,
  80				  struct drm_private_state *state)
  81{
  82	struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(state);
  83
  84	kfree(ctm_state);
  85}
  86
  87static const struct drm_private_state_funcs vc4_ctm_state_funcs = {
  88	.atomic_duplicate_state = vc4_ctm_duplicate_state,
  89	.atomic_destroy_state = vc4_ctm_destroy_state,
  90};
  91
  92static void vc4_ctm_obj_fini(struct drm_device *dev, void *unused)
  93{
  94	struct vc4_dev *vc4 = to_vc4_dev(dev);
  95
  96	drm_atomic_private_obj_fini(&vc4->ctm_manager);
  97}
  98
  99static int vc4_ctm_obj_init(struct vc4_dev *vc4)
 100{
 101	struct vc4_ctm_state *ctm_state;
 102
 103	drm_modeset_lock_init(&vc4->ctm_state_lock);
 104
 105	ctm_state = kzalloc(sizeof(*ctm_state), GFP_KERNEL);
 106	if (!ctm_state)
 107		return -ENOMEM;
 108
 109	drm_atomic_private_obj_init(&vc4->base, &vc4->ctm_manager, &ctm_state->base,
 110				    &vc4_ctm_state_funcs);
 111
 112	return drmm_add_action_or_reset(&vc4->base, vc4_ctm_obj_fini, NULL);
 113}
 114
 115/* Converts a DRM S31.32 value to the HW S0.9 format. */
 116static u16 vc4_ctm_s31_32_to_s0_9(u64 in)
 117{
 118	u16 r;
 119
 120	/* Sign bit. */
 121	r = in & BIT_ULL(63) ? BIT(9) : 0;
 122
 123	if ((in & GENMASK_ULL(62, 32)) > 0) {
 124		/* We have zero integer bits so we can only saturate here. */
 125		r |= GENMASK(8, 0);
 126	} else {
 127		/* Otherwise take the 9 most important fractional bits. */
 128		r |= (in >> 23) & GENMASK(8, 0);
 129	}
 130
 131	return r;
 132}
 133
 134static void
 135vc4_ctm_commit(struct vc4_dev *vc4, struct drm_atomic_state *state)
 136{
 137	struct vc4_hvs *hvs = vc4->hvs;
 138	struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(vc4->ctm_manager.state);
 139	struct drm_color_ctm *ctm = ctm_state->ctm;
 140
 141	if (ctm_state->fifo) {
 142		HVS_WRITE(SCALER_OLEDCOEF2,
 143			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[0]),
 144					SCALER_OLEDCOEF2_R_TO_R) |
 145			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[3]),
 146					SCALER_OLEDCOEF2_R_TO_G) |
 147			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[6]),
 148					SCALER_OLEDCOEF2_R_TO_B));
 149		HVS_WRITE(SCALER_OLEDCOEF1,
 150			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[1]),
 151					SCALER_OLEDCOEF1_G_TO_R) |
 152			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[4]),
 153					SCALER_OLEDCOEF1_G_TO_G) |
 154			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[7]),
 155					SCALER_OLEDCOEF1_G_TO_B));
 156		HVS_WRITE(SCALER_OLEDCOEF0,
 157			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[2]),
 158					SCALER_OLEDCOEF0_B_TO_R) |
 159			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[5]),
 160					SCALER_OLEDCOEF0_B_TO_G) |
 161			  VC4_SET_FIELD(vc4_ctm_s31_32_to_s0_9(ctm->matrix[8]),
 162					SCALER_OLEDCOEF0_B_TO_B));
 163	}
 164
 165	HVS_WRITE(SCALER_OLEDOFFS,
 166		  VC4_SET_FIELD(ctm_state->fifo, SCALER_OLEDOFFS_DISPFIFO));
 167}
 168
 169struct vc4_hvs_state *
 170vc4_hvs_get_new_global_state(const struct drm_atomic_state *state)
 171{
 172	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 173	struct drm_private_state *priv_state;
 174
 175	priv_state = drm_atomic_get_new_private_obj_state(state, &vc4->hvs_channels);
 176	if (!priv_state)
 177		return ERR_PTR(-EINVAL);
 178
 179	return to_vc4_hvs_state(priv_state);
 180}
 
 181
 182struct vc4_hvs_state *
 183vc4_hvs_get_old_global_state(const struct drm_atomic_state *state)
 184{
 185	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 186	struct drm_private_state *priv_state;
 187
 188	priv_state = drm_atomic_get_old_private_obj_state(state, &vc4->hvs_channels);
 189	if (!priv_state)
 190		return ERR_PTR(-EINVAL);
 191
 192	return to_vc4_hvs_state(priv_state);
 193}
 194
 195struct vc4_hvs_state *
 196vc4_hvs_get_global_state(struct drm_atomic_state *state)
 197{
 198	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 199	struct drm_private_state *priv_state;
 200
 201	priv_state = drm_atomic_get_private_obj_state(state, &vc4->hvs_channels);
 202	if (IS_ERR(priv_state))
 203		return ERR_CAST(priv_state);
 204
 205	return to_vc4_hvs_state(priv_state);
 206}
 207
 208static void vc4_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 209				     struct drm_atomic_state *state)
 210{
 211	struct vc4_hvs *hvs = vc4->hvs;
 212	struct drm_crtc_state *crtc_state;
 213	struct drm_crtc *crtc;
 214	unsigned int i;
 215
 216	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 217		struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 218		struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
 219		u32 dispctrl;
 220		u32 dsp3_mux;
 221
 222		if (!crtc_state->active)
 223			continue;
 224
 225		if (vc4_state->assigned_channel != 2)
 226			continue;
 227
 228		/*
 229		 * SCALER_DISPCTRL_DSP3 = X, where X < 2 means 'connect DSP3 to
 230		 * FIFO X'.
 231		 * SCALER_DISPCTRL_DSP3 = 3 means 'disable DSP 3'.
 232		 *
 233		 * DSP3 is connected to FIFO2 unless the transposer is
 234		 * enabled. In this case, FIFO 2 is directly accessed by the
 235		 * TXP IP, and we need to disable the FIFO2 -> pixelvalve1
 236		 * route.
 237		 */
 238		if (vc4_crtc->feeds_txp)
 239			dsp3_mux = VC4_SET_FIELD(3, SCALER_DISPCTRL_DSP3_MUX);
 240		else
 241			dsp3_mux = VC4_SET_FIELD(2, SCALER_DISPCTRL_DSP3_MUX);
 242
 243		dispctrl = HVS_READ(SCALER_DISPCTRL) &
 244			   ~SCALER_DISPCTRL_DSP3_MUX_MASK;
 245		HVS_WRITE(SCALER_DISPCTRL, dispctrl | dsp3_mux);
 246	}
 247}
 248
 249static void vc5_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 250				     struct drm_atomic_state *state)
 251{
 252	struct vc4_hvs *hvs = vc4->hvs;
 253	struct drm_crtc_state *crtc_state;
 254	struct drm_crtc *crtc;
 255	unsigned char mux;
 256	unsigned int i;
 257	u32 reg;
 258
 259	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 260		struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
 261		struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 262		unsigned int channel = vc4_state->assigned_channel;
 263
 264		if (!vc4_state->update_muxing)
 265			continue;
 266
 267		switch (vc4_crtc->data->hvs_output) {
 268		case 2:
 269			drm_WARN_ON(&vc4->base,
 270				    VC4_GET_FIELD(HVS_READ(SCALER_DISPCTRL),
 271						  SCALER_DISPCTRL_DSP3_MUX) == channel);
 272
 273			mux = (channel == 2) ? 0 : 1;
 274			reg = HVS_READ(SCALER_DISPECTRL);
 275			HVS_WRITE(SCALER_DISPECTRL,
 276				  (reg & ~SCALER_DISPECTRL_DSP2_MUX_MASK) |
 277				  VC4_SET_FIELD(mux, SCALER_DISPECTRL_DSP2_MUX));
 278			break;
 279
 280		case 3:
 281			if (channel == VC4_HVS_CHANNEL_DISABLED)
 282				mux = 3;
 283			else
 284				mux = channel;
 285
 286			reg = HVS_READ(SCALER_DISPCTRL);
 287			HVS_WRITE(SCALER_DISPCTRL,
 288				  (reg & ~SCALER_DISPCTRL_DSP3_MUX_MASK) |
 289				  VC4_SET_FIELD(mux, SCALER_DISPCTRL_DSP3_MUX));
 290			break;
 291
 292		case 4:
 293			if (channel == VC4_HVS_CHANNEL_DISABLED)
 294				mux = 3;
 295			else
 296				mux = channel;
 297
 298			reg = HVS_READ(SCALER_DISPEOLN);
 299			HVS_WRITE(SCALER_DISPEOLN,
 300				  (reg & ~SCALER_DISPEOLN_DSP4_MUX_MASK) |
 301				  VC4_SET_FIELD(mux, SCALER_DISPEOLN_DSP4_MUX));
 302
 303			break;
 304
 305		case 5:
 306			if (channel == VC4_HVS_CHANNEL_DISABLED)
 307				mux = 3;
 308			else
 309				mux = channel;
 310
 311			reg = HVS_READ(SCALER_DISPDITHER);
 312			HVS_WRITE(SCALER_DISPDITHER,
 313				  (reg & ~SCALER_DISPDITHER_DSP5_MUX_MASK) |
 314				  VC4_SET_FIELD(mux, SCALER_DISPDITHER_DSP5_MUX));
 315			break;
 316
 317		default:
 318			break;
 319		}
 320	}
 
 
 321}
 322
 323static void vc4_atomic_commit_tail(struct drm_atomic_state *state)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 324{
 325	struct drm_device *dev = state->dev;
 326	struct vc4_dev *vc4 = to_vc4_dev(dev);
 327	struct vc4_hvs *hvs = vc4->hvs;
 328	struct drm_crtc_state *new_crtc_state;
 329	struct vc4_hvs_state *new_hvs_state;
 330	struct drm_crtc *crtc;
 331	struct vc4_hvs_state *old_hvs_state;
 332	unsigned int channel;
 333	int i;
 334
 335	old_hvs_state = vc4_hvs_get_old_global_state(state);
 336	if (WARN_ON(IS_ERR(old_hvs_state)))
 337		return;
 338
 339	new_hvs_state = vc4_hvs_get_new_global_state(state);
 340	if (WARN_ON(IS_ERR(new_hvs_state)))
 341		return;
 342
 343	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
 344		struct vc4_crtc_state *vc4_crtc_state;
 345
 346		if (!new_crtc_state->commit)
 347			continue;
 348
 349		vc4_crtc_state = to_vc4_crtc_state(new_crtc_state);
 350		vc4_hvs_mask_underrun(hvs, vc4_crtc_state->assigned_channel);
 351	}
 352
 353	for (channel = 0; channel < HVS_NUM_CHANNELS; channel++) {
 354		struct drm_crtc_commit *commit;
 355		int ret;
 356
 357		if (!old_hvs_state->fifo_state[channel].in_use)
 358			continue;
 359
 360		commit = old_hvs_state->fifo_state[channel].pending_commit;
 361		if (!commit)
 362			continue;
 363
 364		ret = drm_crtc_commit_wait(commit);
 
 365		if (ret)
 366			drm_err(dev, "Timed out waiting for commit\n");
 367
 368		drm_crtc_commit_put(commit);
 369		old_hvs_state->fifo_state[channel].pending_commit = NULL;
 370	}
 371
 372	if (vc4->gen == VC4_GEN_5) {
 373		unsigned long state_rate = max(old_hvs_state->core_clock_rate,
 374					       new_hvs_state->core_clock_rate);
 375		unsigned long core_rate = clamp_t(unsigned long, state_rate,
 376						  500000000, hvs->max_core_rate);
 377
 378		drm_dbg(dev, "Raising the core clock at %lu Hz\n", core_rate);
 379
 380		/*
 381		 * Do a temporary request on the core clock during the
 382		 * modeset.
 383		 */
 384		WARN_ON(clk_set_min_rate(hvs->core_clk, core_rate));
 385	}
 386
 387	drm_atomic_helper_commit_modeset_disables(dev, state);
 388
 389	vc4_ctm_commit(vc4, state);
 390
 391	if (vc4->gen == VC4_GEN_5)
 392		vc5_hvs_pv_muxing_commit(vc4, state);
 393	else
 394		vc4_hvs_pv_muxing_commit(vc4, state);
 395
 396	drm_atomic_helper_commit_planes(dev, state,
 397					DRM_PLANE_COMMIT_ACTIVE_ONLY);
 398
 399	drm_atomic_helper_commit_modeset_enables(dev, state);
 400
 401	drm_atomic_helper_fake_vblank(state);
 402
 403	drm_atomic_helper_commit_hw_done(state);
 
 404
 405	drm_atomic_helper_wait_for_flip_done(dev, state);
 
 
 
 
 
 
 
 
 406
 407	drm_atomic_helper_cleanup_planes(dev, state);
 408
 409	if (vc4->gen == VC4_GEN_5) {
 410		unsigned long core_rate = min_t(unsigned long,
 411						hvs->max_core_rate,
 412						new_hvs_state->core_clock_rate);
 413
 414		drm_dbg(dev, "Running the core clock at %lu Hz\n", core_rate);
 415
 416		/*
 417		 * Request a clock rate based on the current HVS
 418		 * requirements.
 419		 */
 420		WARN_ON(clk_set_min_rate(hvs->core_clk, core_rate));
 421
 422		drm_dbg(dev, "Core clock actual rate: %lu Hz\n",
 423			clk_get_rate(hvs->core_clk));
 
 
 424	}
 425}
 426
 427static int vc4_atomic_commit_setup(struct drm_atomic_state *state)
 428{
 429	struct drm_crtc_state *crtc_state;
 430	struct vc4_hvs_state *hvs_state;
 431	struct drm_crtc *crtc;
 432	unsigned int i;
 
 
 433
 434	hvs_state = vc4_hvs_get_new_global_state(state);
 435	if (WARN_ON(IS_ERR(hvs_state)))
 436		return PTR_ERR(hvs_state);
 437
 438	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 439		struct vc4_crtc_state *vc4_crtc_state =
 440			to_vc4_crtc_state(crtc_state);
 441		unsigned int channel =
 442			vc4_crtc_state->assigned_channel;
 443
 444		if (channel == VC4_HVS_CHANNEL_DISABLED)
 445			continue;
 446
 447		if (!hvs_state->fifo_state[channel].in_use)
 448			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 449
 450		hvs_state->fifo_state[channel].pending_commit =
 451			drm_crtc_commit_get(crtc_state->commit);
 452	}
 
 
 453
 454	return 0;
 455}
 456
 457static struct drm_framebuffer *vc4_fb_create(struct drm_device *dev,
 458					     struct drm_file *file_priv,
 459					     const struct drm_mode_fb_cmd2 *mode_cmd)
 460{
 461	struct vc4_dev *vc4 = to_vc4_dev(dev);
 462	struct drm_mode_fb_cmd2 mode_cmd_local;
 463
 464	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 465		return ERR_PTR(-ENODEV);
 466
 467	/* If the user didn't specify a modifier, use the
 468	 * vc4_set_tiling_ioctl() state for the BO.
 469	 */
 470	if (!(mode_cmd->flags & DRM_MODE_FB_MODIFIERS)) {
 471		struct drm_gem_object *gem_obj;
 472		struct vc4_bo *bo;
 473
 474		gem_obj = drm_gem_object_lookup(file_priv,
 475						mode_cmd->handles[0]);
 476		if (!gem_obj) {
 477			DRM_DEBUG("Failed to look up GEM BO %d\n",
 478				  mode_cmd->handles[0]);
 479			return ERR_PTR(-ENOENT);
 480		}
 481		bo = to_vc4_bo(gem_obj);
 482
 483		mode_cmd_local = *mode_cmd;
 484
 485		if (bo->t_format) {
 486			mode_cmd_local.modifier[0] =
 487				DRM_FORMAT_MOD_BROADCOM_VC4_T_TILED;
 488		} else {
 489			mode_cmd_local.modifier[0] = DRM_FORMAT_MOD_NONE;
 490		}
 491
 492		drm_gem_object_put(gem_obj);
 493
 494		mode_cmd = &mode_cmd_local;
 495	}
 496
 497	return drm_gem_fb_create(dev, file_priv, mode_cmd);
 498}
 499
 500/* Our CTM has some peculiar limitations: we can only enable it for one CRTC
 501 * at a time and the HW only supports S0.9 scalars. To account for the latter,
 502 * we don't allow userland to set a CTM that we have no hope of approximating.
 503 */
 504static int
 505vc4_ctm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 506{
 507	struct vc4_dev *vc4 = to_vc4_dev(dev);
 508	struct vc4_ctm_state *ctm_state = NULL;
 509	struct drm_crtc *crtc;
 510	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 511	struct drm_color_ctm *ctm;
 512	int i;
 513
 514	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 515		/* CTM is being disabled. */
 516		if (!new_crtc_state->ctm && old_crtc_state->ctm) {
 517			ctm_state = vc4_get_ctm_state(state, &vc4->ctm_manager);
 518			if (IS_ERR(ctm_state))
 519				return PTR_ERR(ctm_state);
 520			ctm_state->fifo = 0;
 521		}
 522	}
 523
 524	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 525		if (new_crtc_state->ctm == old_crtc_state->ctm)
 526			continue;
 527
 528		if (!ctm_state) {
 529			ctm_state = vc4_get_ctm_state(state, &vc4->ctm_manager);
 530			if (IS_ERR(ctm_state))
 531				return PTR_ERR(ctm_state);
 532		}
 533
 534		/* CTM is being enabled or the matrix changed. */
 535		if (new_crtc_state->ctm) {
 536			struct vc4_crtc_state *vc4_crtc_state =
 537				to_vc4_crtc_state(new_crtc_state);
 538
 539			/* fifo is 1-based since 0 disables CTM. */
 540			int fifo = vc4_crtc_state->assigned_channel + 1;
 541
 542			/* Check userland isn't trying to turn on CTM for more
 543			 * than one CRTC at a time.
 544			 */
 545			if (ctm_state->fifo && ctm_state->fifo != fifo) {
 546				DRM_DEBUG_DRIVER("Too many CTM configured\n");
 547				return -EINVAL;
 548			}
 549
 550			/* Check we can approximate the specified CTM.
 551			 * We disallow scalars |c| > 1.0 since the HW has
 552			 * no integer bits.
 553			 */
 554			ctm = new_crtc_state->ctm->data;
 555			for (i = 0; i < ARRAY_SIZE(ctm->matrix); i++) {
 556				u64 val = ctm->matrix[i];
 557
 558				val &= ~BIT_ULL(63);
 559				if (val > BIT_ULL(32))
 560					return -EINVAL;
 561			}
 562
 563			ctm_state->fifo = fifo;
 564			ctm_state->ctm = ctm;
 565		}
 566	}
 567
 568	return 0;
 569}
 570
 571static int vc4_load_tracker_atomic_check(struct drm_atomic_state *state)
 572{
 573	struct drm_plane_state *old_plane_state, *new_plane_state;
 574	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 575	struct vc4_load_tracker_state *load_state;
 576	struct drm_private_state *priv_state;
 577	struct drm_plane *plane;
 578	int i;
 579
 580	priv_state = drm_atomic_get_private_obj_state(state,
 581						      &vc4->load_tracker);
 582	if (IS_ERR(priv_state))
 583		return PTR_ERR(priv_state);
 584
 585	load_state = to_vc4_load_tracker_state(priv_state);
 586	for_each_oldnew_plane_in_state(state, plane, old_plane_state,
 587				       new_plane_state, i) {
 588		struct vc4_plane_state *vc4_plane_state;
 589
 590		if (old_plane_state->fb && old_plane_state->crtc) {
 591			vc4_plane_state = to_vc4_plane_state(old_plane_state);
 592			load_state->membus_load -= vc4_plane_state->membus_load;
 593			load_state->hvs_load -= vc4_plane_state->hvs_load;
 594		}
 595
 596		if (new_plane_state->fb && new_plane_state->crtc) {
 597			vc4_plane_state = to_vc4_plane_state(new_plane_state);
 598			load_state->membus_load += vc4_plane_state->membus_load;
 599			load_state->hvs_load += vc4_plane_state->hvs_load;
 600		}
 601	}
 602
 603	/* Don't check the load when the tracker is disabled. */
 604	if (!vc4->load_tracker_enabled)
 605		return 0;
 606
 607	/* The absolute limit is 2Gbyte/sec, but let's take a margin to let
 608	 * the system work when other blocks are accessing the memory.
 609	 */
 610	if (load_state->membus_load > SZ_1G + SZ_512M)
 611		return -ENOSPC;
 612
 613	/* HVS clock is supposed to run @ 250Mhz, let's take a margin and
 614	 * consider the maximum number of cycles is 240M.
 615	 */
 616	if (load_state->hvs_load > 240000000ULL)
 617		return -ENOSPC;
 618
 619	return 0;
 620}
 621
 622static struct drm_private_state *
 623vc4_load_tracker_duplicate_state(struct drm_private_obj *obj)
 624{
 625	struct vc4_load_tracker_state *state;
 626
 627	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
 628	if (!state)
 629		return NULL;
 630
 631	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 632
 633	return &state->base;
 634}
 635
 636static void vc4_load_tracker_destroy_state(struct drm_private_obj *obj,
 637					   struct drm_private_state *state)
 638{
 639	struct vc4_load_tracker_state *load_state;
 640
 641	load_state = to_vc4_load_tracker_state(state);
 642	kfree(load_state);
 643}
 644
 645static const struct drm_private_state_funcs vc4_load_tracker_state_funcs = {
 646	.atomic_duplicate_state = vc4_load_tracker_duplicate_state,
 647	.atomic_destroy_state = vc4_load_tracker_destroy_state,
 648};
 649
 650static void vc4_load_tracker_obj_fini(struct drm_device *dev, void *unused)
 651{
 652	struct vc4_dev *vc4 = to_vc4_dev(dev);
 653
 654	drm_atomic_private_obj_fini(&vc4->load_tracker);
 655}
 656
 657static int vc4_load_tracker_obj_init(struct vc4_dev *vc4)
 658{
 659	struct vc4_load_tracker_state *load_state;
 660
 661	load_state = kzalloc(sizeof(*load_state), GFP_KERNEL);
 662	if (!load_state)
 663		return -ENOMEM;
 664
 665	drm_atomic_private_obj_init(&vc4->base, &vc4->load_tracker,
 666				    &load_state->base,
 667				    &vc4_load_tracker_state_funcs);
 668
 669	return drmm_add_action_or_reset(&vc4->base, vc4_load_tracker_obj_fini, NULL);
 670}
 671
 672static struct drm_private_state *
 673vc4_hvs_channels_duplicate_state(struct drm_private_obj *obj)
 674{
 675	struct vc4_hvs_state *old_state = to_vc4_hvs_state(obj->state);
 676	struct vc4_hvs_state *state;
 677	unsigned int i;
 678
 679	state = kzalloc(sizeof(*state), GFP_KERNEL);
 680	if (!state)
 681		return NULL;
 682
 683	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 684
 685	for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 686		state->fifo_state[i].in_use = old_state->fifo_state[i].in_use;
 687		state->fifo_state[i].fifo_load = old_state->fifo_state[i].fifo_load;
 688	}
 689
 690	state->core_clock_rate = old_state->core_clock_rate;
 691
 692	return &state->base;
 693}
 694
 695static void vc4_hvs_channels_destroy_state(struct drm_private_obj *obj,
 696					   struct drm_private_state *state)
 697{
 698	struct vc4_hvs_state *hvs_state = to_vc4_hvs_state(state);
 699	unsigned int i;
 700
 701	for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 702		if (!hvs_state->fifo_state[i].pending_commit)
 703			continue;
 704
 705		drm_crtc_commit_put(hvs_state->fifo_state[i].pending_commit);
 706	}
 707
 708	kfree(hvs_state);
 709}
 710
 711static void vc4_hvs_channels_print_state(struct drm_printer *p,
 712					 const struct drm_private_state *state)
 713{
 714	const struct vc4_hvs_state *hvs_state = to_vc4_hvs_state(state);
 715	unsigned int i;
 716
 717	drm_printf(p, "HVS State\n");
 718	drm_printf(p, "\tCore Clock Rate: %lu\n", hvs_state->core_clock_rate);
 719
 720	for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 721		drm_printf(p, "\tChannel %d\n", i);
 722		drm_printf(p, "\t\tin use=%d\n", hvs_state->fifo_state[i].in_use);
 723		drm_printf(p, "\t\tload=%lu\n", hvs_state->fifo_state[i].fifo_load);
 724	}
 725}
 726
 727static const struct drm_private_state_funcs vc4_hvs_state_funcs = {
 728	.atomic_duplicate_state = vc4_hvs_channels_duplicate_state,
 729	.atomic_destroy_state = vc4_hvs_channels_destroy_state,
 730	.atomic_print_state = vc4_hvs_channels_print_state,
 731};
 732
 733static void vc4_hvs_channels_obj_fini(struct drm_device *dev, void *unused)
 734{
 735	struct vc4_dev *vc4 = to_vc4_dev(dev);
 736
 737	drm_atomic_private_obj_fini(&vc4->hvs_channels);
 738}
 739
 740static int vc4_hvs_channels_obj_init(struct vc4_dev *vc4)
 741{
 742	struct vc4_hvs_state *state;
 743
 744	state = kzalloc(sizeof(*state), GFP_KERNEL);
 745	if (!state)
 746		return -ENOMEM;
 747
 748	drm_atomic_private_obj_init(&vc4->base, &vc4->hvs_channels,
 749				    &state->base,
 750				    &vc4_hvs_state_funcs);
 751
 752	return drmm_add_action_or_reset(&vc4->base, vc4_hvs_channels_obj_fini, NULL);
 753}
 754
 755static int cmp_vc4_crtc_hvs_output(const void *a, const void *b)
 756{
 757	const struct vc4_crtc *crtc_a =
 758		to_vc4_crtc(*(const struct drm_crtc **)a);
 759	const struct vc4_crtc_data *data_a =
 760		vc4_crtc_to_vc4_crtc_data(crtc_a);
 761	const struct vc4_crtc *crtc_b =
 762		to_vc4_crtc(*(const struct drm_crtc **)b);
 763	const struct vc4_crtc_data *data_b =
 764		vc4_crtc_to_vc4_crtc_data(crtc_b);
 765
 766	return data_a->hvs_output - data_b->hvs_output;
 767}
 768
 769/*
 770 * The BCM2711 HVS has up to 7 outputs connected to the pixelvalves and
 771 * the TXP (and therefore all the CRTCs found on that platform).
 772 *
 773 * The naive (and our initial) implementation would just iterate over
 774 * all the active CRTCs, try to find a suitable FIFO, and then remove it
 775 * from the pool of available FIFOs. However, there are a few corner
 776 * cases that need to be considered:
 777 *
 778 * - When running in a dual-display setup (so with two CRTCs involved),
 779 *   we can update the state of a single CRTC (for example by changing
 780 *   its mode using xrandr under X11) without affecting the other. In
 781 *   this case, the other CRTC wouldn't be in the state at all, so we
 782 *   need to consider all the running CRTCs in the DRM device to assign
 783 *   a FIFO, not just the one in the state.
 784 *
 785 * - To fix the above, we can't use drm_atomic_get_crtc_state on all
 786 *   enabled CRTCs to pull their CRTC state into the global state, since
 787 *   a page flip would start considering their vblank to complete. Since
 788 *   we don't have a guarantee that they are actually active, that
 789 *   vblank might never happen, and shouldn't even be considered if we
 790 *   want to do a page flip on a single CRTC. That can be tested by
 791 *   doing a modetest -v first on HDMI1 and then on HDMI0.
 792 *
 793 * - Since we need the pixelvalve to be disabled and enabled back when
 794 *   the FIFO is changed, we should keep the FIFO assigned for as long
 795 *   as the CRTC is enabled, only considering it free again once that
 796 *   CRTC has been disabled. This can be tested by booting X11 on a
 797 *   single display, and changing the resolution down and then back up.
 798 */
 799static int vc4_pv_muxing_atomic_check(struct drm_device *dev,
 800				      struct drm_atomic_state *state)
 801{
 802	struct vc4_hvs_state *hvs_new_state;
 803	struct drm_crtc **sorted_crtcs;
 804	struct drm_crtc *crtc;
 805	unsigned int unassigned_channels = 0;
 806	unsigned int i;
 807	int ret;
 808
 809	hvs_new_state = vc4_hvs_get_global_state(state);
 810	if (IS_ERR(hvs_new_state))
 811		return PTR_ERR(hvs_new_state);
 812
 813	for (i = 0; i < ARRAY_SIZE(hvs_new_state->fifo_state); i++)
 814		if (!hvs_new_state->fifo_state[i].in_use)
 815			unassigned_channels |= BIT(i);
 816
 817	/*
 818	 * The problem we have to solve here is that we have up to 7
 819	 * encoders, connected to up to 6 CRTCs.
 820	 *
 821	 * Those CRTCs, depending on the instance, can be routed to 1, 2
 822	 * or 3 HVS FIFOs, and we need to set the muxing between FIFOs and
 823	 * outputs in the HVS accordingly.
 824	 *
 825	 * It would be pretty hard to come up with an algorithm that
 826	 * would generically solve this. However, the current routing
 827	 * trees we support allow us to simplify a bit the problem.
 828	 *
 829	 * Indeed, with the current supported layouts, if we try to
 830	 * assign in the ascending crtc index order the FIFOs, we can't
 831	 * fall into the situation where an earlier CRTC that had
 832	 * multiple routes is assigned one that was the only option for
 833	 * a later CRTC.
 834	 *
 835	 * If the layout changes and doesn't give us that in the future,
 836	 * we will need to have something smarter, but it works so far.
 837	 */
 838	sorted_crtcs = kmalloc_array(dev->num_crtcs, sizeof(*sorted_crtcs), GFP_KERNEL);
 839	if (!sorted_crtcs)
 840		return -ENOMEM;
 841
 842	i = 0;
 843	drm_for_each_crtc(crtc, dev)
 844		sorted_crtcs[i++] = crtc;
 845
 846	sort(sorted_crtcs, i, sizeof(*sorted_crtcs), cmp_vc4_crtc_hvs_output, NULL);
 847
 848	for (i = 0; i < dev->num_crtcs; i++) {
 849		struct vc4_crtc_state *old_vc4_crtc_state, *new_vc4_crtc_state;
 850		struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 851		struct vc4_crtc *vc4_crtc;
 852		unsigned int matching_channels;
 853		unsigned int channel;
 854
 855		crtc = sorted_crtcs[i];
 856		if (!crtc)
 857			continue;
 858		vc4_crtc = to_vc4_crtc(crtc);
 859
 860		old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
 861		if (!old_crtc_state)
 862			continue;
 863		old_vc4_crtc_state = to_vc4_crtc_state(old_crtc_state);
 864
 865		new_crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 866		if (!new_crtc_state)
 867			continue;
 868		new_vc4_crtc_state = to_vc4_crtc_state(new_crtc_state);
 869
 870		drm_dbg(dev, "%s: Trying to find a channel.\n", crtc->name);
 871
 872		/* Nothing to do here, let's skip it */
 873		if (old_crtc_state->enable == new_crtc_state->enable) {
 874			if (new_crtc_state->enable)
 875				drm_dbg(dev, "%s: Already enabled, reusing channel %d.\n",
 876					crtc->name, new_vc4_crtc_state->assigned_channel);
 877			else
 878				drm_dbg(dev, "%s: Disabled, ignoring.\n", crtc->name);
 879
 880			continue;
 881		}
 882
 883		/* Muxing will need to be modified, mark it as such */
 884		new_vc4_crtc_state->update_muxing = true;
 885
 886		/* If we're disabling our CRTC, we put back our channel */
 887		if (!new_crtc_state->enable) {
 888			channel = old_vc4_crtc_state->assigned_channel;
 889
 890			drm_dbg(dev, "%s: Disabling, Freeing channel %d\n",
 891				crtc->name, channel);
 892
 893			hvs_new_state->fifo_state[channel].in_use = false;
 894			new_vc4_crtc_state->assigned_channel = VC4_HVS_CHANNEL_DISABLED;
 895			continue;
 896		}
 897
 898		matching_channels = unassigned_channels & vc4_crtc->data->hvs_available_channels;
 899		if (!matching_channels) {
 900			ret = -EINVAL;
 901			goto err_free_crtc_array;
 902		}
 903
 904		channel = ffs(matching_channels) - 1;
 905
 906		drm_dbg(dev, "Assigned HVS channel %d to CRTC %s\n", channel, crtc->name);
 907		new_vc4_crtc_state->assigned_channel = channel;
 908		unassigned_channels &= ~BIT(channel);
 909		hvs_new_state->fifo_state[channel].in_use = true;
 910	}
 911
 912	kfree(sorted_crtcs);
 913	return 0;
 914
 915err_free_crtc_array:
 916	kfree(sorted_crtcs);
 917	return ret;
 918}
 919
 920static int
 921vc4_core_clock_atomic_check(struct drm_atomic_state *state)
 922{
 923	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 924	struct drm_private_state *priv_state;
 925	struct vc4_hvs_state *hvs_new_state;
 926	struct vc4_load_tracker_state *load_state;
 927	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 928	struct drm_crtc *crtc;
 929	unsigned int num_outputs;
 930	unsigned long pixel_rate;
 931	unsigned long cob_rate;
 932	unsigned int i;
 933
 934	priv_state = drm_atomic_get_private_obj_state(state,
 935						      &vc4->load_tracker);
 936	if (IS_ERR(priv_state))
 937		return PTR_ERR(priv_state);
 938
 939	load_state = to_vc4_load_tracker_state(priv_state);
 940
 941	hvs_new_state = vc4_hvs_get_global_state(state);
 942	if (IS_ERR(hvs_new_state))
 943		return PTR_ERR(hvs_new_state);
 944
 945	for_each_oldnew_crtc_in_state(state, crtc,
 946				      old_crtc_state,
 947				      new_crtc_state,
 948				      i) {
 949		if (old_crtc_state->active) {
 950			struct vc4_crtc_state *old_vc4_state =
 951				to_vc4_crtc_state(old_crtc_state);
 952			unsigned int channel = old_vc4_state->assigned_channel;
 953
 954			hvs_new_state->fifo_state[channel].fifo_load = 0;
 955		}
 956
 957		if (new_crtc_state->active) {
 958			struct vc4_crtc_state *new_vc4_state =
 959				to_vc4_crtc_state(new_crtc_state);
 960			unsigned int channel = new_vc4_state->assigned_channel;
 961
 962			hvs_new_state->fifo_state[channel].fifo_load =
 963				new_vc4_state->hvs_load;
 964		}
 965	}
 966
 967	cob_rate = 0;
 968	num_outputs = 0;
 969	for (i = 0; i < HVS_NUM_CHANNELS; i++) {
 970		if (!hvs_new_state->fifo_state[i].in_use)
 971			continue;
 972
 973		num_outputs++;
 974		cob_rate = max_t(unsigned long,
 975				 hvs_new_state->fifo_state[i].fifo_load,
 976				 cob_rate);
 977	}
 978
 979	pixel_rate = load_state->hvs_load;
 980	if (num_outputs > 1) {
 981		pixel_rate = (pixel_rate * 40) / 100;
 982	} else {
 983		pixel_rate = (pixel_rate * 60) / 100;
 984	}
 985
 986	hvs_new_state->core_clock_rate = max(cob_rate, pixel_rate);
 987
 988	return 0;
 989}
 990
 991
 992static int
 993vc4_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 994{
 995	int ret;
 996
 997	ret = vc4_pv_muxing_atomic_check(dev, state);
 998	if (ret)
 999		return ret;
1000
1001	ret = vc4_ctm_atomic_check(dev, state);
1002	if (ret < 0)
1003		return ret;
1004
1005	ret = drm_atomic_helper_check(dev, state);
1006	if (ret)
1007		return ret;
1008
1009	ret = vc4_load_tracker_atomic_check(state);
1010	if (ret)
1011		return ret;
1012
1013	return vc4_core_clock_atomic_check(state);
1014}
1015
1016static struct drm_mode_config_helper_funcs vc4_mode_config_helpers = {
1017	.atomic_commit_setup	= vc4_atomic_commit_setup,
1018	.atomic_commit_tail	= vc4_atomic_commit_tail,
1019};
1020
1021static const struct drm_mode_config_funcs vc4_mode_funcs = {
1022	.atomic_check = vc4_atomic_check,
1023	.atomic_commit = drm_atomic_helper_commit,
1024	.fb_create = vc4_fb_create,
1025};
1026
1027static const struct drm_mode_config_funcs vc5_mode_funcs = {
1028	.atomic_check = vc4_atomic_check,
1029	.atomic_commit = drm_atomic_helper_commit,
1030	.fb_create = drm_gem_fb_create,
1031};
1032
1033int vc4_kms_load(struct drm_device *dev)
1034{
1035	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
 
1036	int ret;
1037
1038	/*
1039	 * The limits enforced by the load tracker aren't relevant for
1040	 * the BCM2711, but the load tracker computations are used for
1041	 * the core clock rate calculation.
1042	 */
1043	if (vc4->gen == VC4_GEN_4) {
1044		/* Start with the load tracker enabled. Can be
1045		 * disabled through the debugfs load_tracker file.
1046		 */
1047		vc4->load_tracker_enabled = true;
1048	}
1049
1050	/* Set support for vblank irq fast disable, before drm_vblank_init() */
1051	dev->vblank_disable_immediate = true;
1052
 
1053	ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
1054	if (ret < 0) {
1055		dev_err(dev->dev, "failed to initialize vblank\n");
1056		return ret;
1057	}
1058
1059	if (vc4->gen == VC4_GEN_5) {
1060		dev->mode_config.max_width = 7680;
1061		dev->mode_config.max_height = 7680;
1062	} else {
1063		dev->mode_config.max_width = 2048;
1064		dev->mode_config.max_height = 2048;
1065	}
1066
1067	dev->mode_config.funcs = (vc4->gen > VC4_GEN_4) ? &vc5_mode_funcs : &vc4_mode_funcs;
1068	dev->mode_config.helper_private = &vc4_mode_config_helpers;
1069	dev->mode_config.preferred_depth = 24;
1070	dev->mode_config.async_page_flip = true;
1071	dev->mode_config.normalize_zpos = true;
1072
1073	ret = vc4_ctm_obj_init(vc4);
1074	if (ret)
1075		return ret;
1076
1077	ret = vc4_load_tracker_obj_init(vc4);
1078	if (ret)
1079		return ret;
1080
1081	ret = vc4_hvs_channels_obj_init(vc4);
1082	if (ret)
1083		return ret;
 
 
 
 
 
 
 
 
1084
1085	drm_mode_config_reset(dev);
1086
1087	drm_kms_helper_poll_init(dev);
1088
1089	return 0;
1090}