1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Tegra host1x Interrupt Management
  4 *
  5 * Copyright (c) 2010-2013, NVIDIA Corporation.
  6 */
  7
  8#include <linux/clk.h>
  9#include <linux/interrupt.h>
 10#include <linux/slab.h>
 11#include <linux/irq.h>
 12
 13#include <trace/events/host1x.h>
 14#include "channel.h"
 15#include "dev.h"
 16#include "intr.h"
 17
 18/* Wait list management */
 19
 20enum waitlist_state {
 21	WLS_PENDING,
 22	WLS_REMOVED,
 23	WLS_CANCELLED,
 24	WLS_HANDLED
 25};
 26
 27static void waiter_release(struct kref *kref)
 28{
 29	kfree(container_of(kref, struct host1x_waitlist, refcount));
 30}
 31
 32/*
 33 * add a waiter to a waiter queue, sorted by threshold
 34 * returns true if it was added at the head of the queue
 35 */
 36static bool add_waiter_to_queue(struct host1x_waitlist *waiter,
 37				struct list_head *queue)
 38{
 39	struct host1x_waitlist *pos;
 40	u32 thresh = waiter->thresh;
 41
 42	list_for_each_entry_reverse(pos, queue, list)
 43		if ((s32)(pos->thresh - thresh) <= 0) {
 44			list_add(&waiter->list, &pos->list);
 45			return false;
 46		}
 47
 48	list_add(&waiter->list, queue);
 49	return true;
 50}
 51
 52/*
 53 * run through a waiter queue for a single sync point ID
 54 * and gather all completed waiters into lists by actions
 55 */
 56static void remove_completed_waiters(struct list_head *head, u32 sync,
 57			struct list_head completed[HOST1X_INTR_ACTION_COUNT])
 58{
 59	struct list_head *dest;
 60	struct host1x_waitlist *waiter, *next, *prev;
 61
 62	list_for_each_entry_safe(waiter, next, head, list) {
 63		if ((s32)(waiter->thresh - sync) > 0)
 64			break;
 65
 66		dest = completed + waiter->action;
 67
 68		/* consolidate submit cleanups */
 69		if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&
 70		    !list_empty(dest)) {
 71			prev = list_entry(dest->prev,
 72					  struct host1x_waitlist, list);
 73			if (prev->data == waiter->data) {
 74				prev->count++;
 75				dest = NULL;
 76			}
 77		}
 78
 79		/* PENDING->REMOVED or CANCELLED->HANDLED */
 80		if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
 81			list_del(&waiter->list);
 82			kref_put(&waiter->refcount, waiter_release);
 83		} else
 84			list_move_tail(&waiter->list, dest);
 85	}
 86}
 87
 88static void reset_threshold_interrupt(struct host1x *host,
 89				      struct list_head *head,
 90				      unsigned int id)
 91{
 92	u32 thresh =
 93		list_first_entry(head, struct host1x_waitlist, list)->thresh;
 94
 95	host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
 96	host1x_hw_intr_enable_syncpt_intr(host, id);
 97}
 98
 99static void action_submit_complete(struct host1x_waitlist *waiter)
100{
101	struct host1x_channel *channel = waiter->data;
102
103	host1x_cdma_update(&channel->cdma);
104
105	/*  Add nr_completed to trace */
106	trace_host1x_channel_submit_complete(dev_name(channel->dev),
107					     waiter->count, waiter->thresh);
108}
109
110static void action_wakeup(struct host1x_waitlist *waiter)
111{
112	wait_queue_head_t *wq = waiter->data;
113
114	wake_up(wq);
115}
116
117static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
118{
119	wait_queue_head_t *wq = waiter->data;
120
121	wake_up_interruptible(wq);
122}
123
124typedef void (*action_handler)(struct host1x_waitlist *waiter);
125
126static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
127	action_submit_complete,
128	action_wakeup,
129	action_wakeup_interruptible,
130};
131
132static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
133{
134	struct list_head *head = completed;
135	unsigned int i;
136
137	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
138		action_handler handler = action_handlers[i];
139		struct host1x_waitlist *waiter, *next;
140
141		list_for_each_entry_safe(waiter, next, head, list) {
142			list_del(&waiter->list);
143			handler(waiter);
144			WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=
145				WLS_REMOVED);
146			kref_put(&waiter->refcount, waiter_release);
147		}
148	}
149}
150
151/*
152 * Remove & handle all waiters that have completed for the given syncpt
153 */
154static int process_wait_list(struct host1x *host,
155			     struct host1x_syncpt *syncpt,
156			     u32 threshold)
157{
158	struct list_head completed[HOST1X_INTR_ACTION_COUNT];
159	unsigned int i;
160	int empty;
161
162	for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i)
163		INIT_LIST_HEAD(completed + i);
164
165	spin_lock(&syncpt->intr.lock);
166
167	remove_completed_waiters(&syncpt->intr.wait_head, threshold,
168				 completed);
169
170	empty = list_empty(&syncpt->intr.wait_head);
171	if (empty)
172		host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);
173	else
174		reset_threshold_interrupt(host, &syncpt->intr.wait_head,
175					  syncpt->id);
176
177	spin_unlock(&syncpt->intr.lock);
178
179	run_handlers(completed);
180
181	return empty;
182}
183
184/*
185 * Sync point threshold interrupt service thread function
186 * Handles sync point threshold triggers, in thread context
187 */
188
189static void syncpt_thresh_work(struct work_struct *work)
190{
191	struct host1x_syncpt_intr *syncpt_intr =
192		container_of(work, struct host1x_syncpt_intr, work);
193	struct host1x_syncpt *syncpt =
194		container_of(syncpt_intr, struct host1x_syncpt, intr);
195	unsigned int id = syncpt->id;
196	struct host1x *host = syncpt->host;
197
198	(void)process_wait_list(host, syncpt,
199				host1x_syncpt_load(host->syncpt + id));
200}
201
202int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,
203			   u32 thresh, enum host1x_intr_action action,
204			   void *data, struct host1x_waitlist *waiter,
205			   void **ref)
206{
207	int queue_was_empty;
208
209	if (waiter == NULL) {
210		pr_warn("%s: NULL waiter\n", __func__);
211		return -EINVAL;
212	}
213
214	/* initialize a new waiter */
215	INIT_LIST_HEAD(&waiter->list);
216	kref_init(&waiter->refcount);
217	if (ref)
218		kref_get(&waiter->refcount);
219	waiter->thresh = thresh;
220	waiter->action = action;
221	atomic_set(&waiter->state, WLS_PENDING);
222	waiter->data = data;
223	waiter->count = 1;
224
225	spin_lock(&syncpt->intr.lock);
226
227	queue_was_empty = list_empty(&syncpt->intr.wait_head);
228
229	if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
230		/* added at head of list - new threshold value */
231		host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);
232
233		/* added as first waiter - enable interrupt */
234		if (queue_was_empty)
235			host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);
236	}
237
238	if (ref)
239		*ref = waiter;
240
241	spin_unlock(&syncpt->intr.lock);
242
243	return 0;
244}
245
246void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref,
247			 bool flush)
248{
249	struct host1x_waitlist *waiter = ref;
250	struct host1x_syncpt *syncpt;
251
252	atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED);
253
254	syncpt = host->syncpt + id;
255
256	spin_lock(&syncpt->intr.lock);
257	if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) ==
258	    WLS_CANCELLED) {
259		list_del(&waiter->list);
260		kref_put(&waiter->refcount, waiter_release);
261	}
262	spin_unlock(&syncpt->intr.lock);
263
264	if (flush) {
265		/* Wait until any concurrently executing handler has finished. */
266		while (atomic_read(&waiter->state) != WLS_HANDLED)
267			schedule();
268	}
269
270	kref_put(&waiter->refcount, waiter_release);
271}
272
273int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
274{
275	unsigned int id;
276	u32 nb_pts = host1x_syncpt_nb_pts(host);
277
278	mutex_init(&host->intr_mutex);
279	host->intr_syncpt_irq = irq_sync;
280
281	for (id = 0; id < nb_pts; ++id) {
282		struct host1x_syncpt *syncpt = host->syncpt + id;
283
284		spin_lock_init(&syncpt->intr.lock);
285		INIT_LIST_HEAD(&syncpt->intr.wait_head);
286		snprintf(syncpt->intr.thresh_irq_name,
287			 sizeof(syncpt->intr.thresh_irq_name),
288			 "host1x_sp_%02u", id);
289	}
290
291	host1x_intr_start(host);
292
293	return 0;
294}
295
296void host1x_intr_deinit(struct host1x *host)
297{
298	host1x_intr_stop(host);
299}
300
301void host1x_intr_start(struct host1x *host)
302{
303	u32 hz = clk_get_rate(host->clk);
304	int err;
305
306	mutex_lock(&host->intr_mutex);
307	err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),
308					    syncpt_thresh_work);
309	if (err) {
310		mutex_unlock(&host->intr_mutex);
311		return;
312	}
313	mutex_unlock(&host->intr_mutex);
314}
315
316void host1x_intr_stop(struct host1x *host)
317{
318	unsigned int id;
319	struct host1x_syncpt *syncpt = host->syncpt;
320	u32 nb_pts = host1x_syncpt_nb_pts(host);
321
322	mutex_lock(&host->intr_mutex);
323
324	host1x_hw_intr_disable_all_syncpt_intrs(host);
325
326	for (id = 0; id < nb_pts; ++id) {
327		struct host1x_waitlist *waiter, *next;
328
329		list_for_each_entry_safe(waiter, next,
330			&syncpt[id].intr.wait_head, list) {
331			if (atomic_cmpxchg(&waiter->state,
332			    WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) {
333				list_del(&waiter->list);
334				kref_put(&waiter->refcount, waiter_release);
335			}
336		}
337
338		if (!list_empty(&syncpt[id].intr.wait_head)) {
339			/* output diagnostics */
340			mutex_unlock(&host->intr_mutex);
341			pr_warn("%s cannot stop syncpt intr id=%u\n",
342				__func__, id);
343			return;
344		}
345	}
346
347	host1x_hw_intr_free_syncpt_irq(host);
348
349	mutex_unlock(&host->intr_mutex);
350}