Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2013-2016 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include <linux/dma-fence.h>
8
9#include "msm_drv.h"
10#include "msm_fence.h"
11#include "msm_gpu.h"
12
13static struct msm_gpu *fctx2gpu(struct msm_fence_context *fctx)
14{
15 struct msm_drm_private *priv = fctx->dev->dev_private;
16 return priv->gpu;
17}
18
19static enum hrtimer_restart deadline_timer(struct hrtimer *t)
20{
21 struct msm_fence_context *fctx = container_of(t,
22 struct msm_fence_context, deadline_timer);
23
24 kthread_queue_work(fctx2gpu(fctx)->worker, &fctx->deadline_work);
25
26 return HRTIMER_NORESTART;
27}
28
29static void deadline_work(struct kthread_work *work)
30{
31 struct msm_fence_context *fctx = container_of(work,
32 struct msm_fence_context, deadline_work);
33
34 /* If deadline fence has already passed, nothing to do: */
35 if (msm_fence_completed(fctx, fctx->next_deadline_fence))
36 return;
37
38 msm_devfreq_boost(fctx2gpu(fctx), 2);
39}
40
41
42struct msm_fence_context *
43msm_fence_context_alloc(struct drm_device *dev, volatile uint32_t *fenceptr,
44 const char *name)
45{
46 struct msm_fence_context *fctx;
47 static int index = 0;
48
49 fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
50 if (!fctx)
51 return ERR_PTR(-ENOMEM);
52
53 fctx->dev = dev;
54 strscpy(fctx->name, name, sizeof(fctx->name));
55 fctx->context = dma_fence_context_alloc(1);
56 fctx->index = index++;
57 fctx->fenceptr = fenceptr;
58 spin_lock_init(&fctx->spinlock);
59
60 /*
61 * Start out close to the 32b fence rollover point, so we can
62 * catch bugs with fence comparisons.
63 */
64 fctx->last_fence = 0xffffff00;
65 fctx->completed_fence = fctx->last_fence;
66 *fctx->fenceptr = fctx->last_fence;
67
68 hrtimer_init(&fctx->deadline_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
69 fctx->deadline_timer.function = deadline_timer;
70
71 kthread_init_work(&fctx->deadline_work, deadline_work);
72
73 fctx->next_deadline = ktime_get();
74
75 return fctx;
76}
77
78void msm_fence_context_free(struct msm_fence_context *fctx)
79{
80 kfree(fctx);
81}
82
83bool msm_fence_completed(struct msm_fence_context *fctx, uint32_t fence)
84{
85 /*
86 * Note: Check completed_fence first, as fenceptr is in a write-combine
87 * mapping, so it will be more expensive to read.
88 */
89 return (int32_t)(fctx->completed_fence - fence) >= 0 ||
90 (int32_t)(*fctx->fenceptr - fence) >= 0;
91}
92
93/* called from irq handler and workqueue (in recover path) */
94void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence)
95{
96 unsigned long flags;
97
98 spin_lock_irqsave(&fctx->spinlock, flags);
99 if (fence_after(fence, fctx->completed_fence))
100 fctx->completed_fence = fence;
101 if (msm_fence_completed(fctx, fctx->next_deadline_fence))
102 hrtimer_cancel(&fctx->deadline_timer);
103 spin_unlock_irqrestore(&fctx->spinlock, flags);
104}
105
106struct msm_fence {
107 struct dma_fence base;
108 struct msm_fence_context *fctx;
109};
110
111static inline struct msm_fence *to_msm_fence(struct dma_fence *fence)
112{
113 return container_of(fence, struct msm_fence, base);
114}
115
116static const char *msm_fence_get_driver_name(struct dma_fence *fence)
117{
118 return "msm";
119}
120
121static const char *msm_fence_get_timeline_name(struct dma_fence *fence)
122{
123 struct msm_fence *f = to_msm_fence(fence);
124 return f->fctx->name;
125}
126
127static bool msm_fence_signaled(struct dma_fence *fence)
128{
129 struct msm_fence *f = to_msm_fence(fence);
130 return msm_fence_completed(f->fctx, f->base.seqno);
131}
132
133static void msm_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
134{
135 struct msm_fence *f = to_msm_fence(fence);
136 struct msm_fence_context *fctx = f->fctx;
137 unsigned long flags;
138 ktime_t now;
139
140 spin_lock_irqsave(&fctx->spinlock, flags);
141 now = ktime_get();
142
143 if (ktime_after(now, fctx->next_deadline) ||
144 ktime_before(deadline, fctx->next_deadline)) {
145 fctx->next_deadline = deadline;
146 fctx->next_deadline_fence =
147 max(fctx->next_deadline_fence, (uint32_t)fence->seqno);
148
149 /*
150 * Set timer to trigger boost 3ms before deadline, or
151 * if we are already less than 3ms before the deadline
152 * schedule boost work immediately.
153 */
154 deadline = ktime_sub(deadline, ms_to_ktime(3));
155
156 if (ktime_after(now, deadline)) {
157 kthread_queue_work(fctx2gpu(fctx)->worker,
158 &fctx->deadline_work);
159 } else {
160 hrtimer_start(&fctx->deadline_timer, deadline,
161 HRTIMER_MODE_ABS);
162 }
163 }
164
165 spin_unlock_irqrestore(&fctx->spinlock, flags);
166}
167
168static const struct dma_fence_ops msm_fence_ops = {
169 .get_driver_name = msm_fence_get_driver_name,
170 .get_timeline_name = msm_fence_get_timeline_name,
171 .signaled = msm_fence_signaled,
172 .set_deadline = msm_fence_set_deadline,
173};
174
175struct dma_fence *
176msm_fence_alloc(void)
177{
178 struct msm_fence *f;
179
180 f = kzalloc(sizeof(*f), GFP_KERNEL);
181 if (!f)
182 return ERR_PTR(-ENOMEM);
183
184 return &f->base;
185}
186
187void
188msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx)
189{
190 struct msm_fence *f = to_msm_fence(fence);
191
192 f->fctx = fctx;
193
194 /*
195 * Until this point, the fence was just some pre-allocated memory,
196 * no-one should have taken a reference to it yet.
197 */
198 WARN_ON(kref_read(&fence->refcount));
199
200 dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
201 fctx->context, ++fctx->last_fence);
202}
1/*
2 * Copyright (C) 2013-2016 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/dma-fence.h>
19
20#include "msm_drv.h"
21#include "msm_fence.h"
22
23
24struct msm_fence_context *
25msm_fence_context_alloc(struct drm_device *dev, const char *name)
26{
27 struct msm_fence_context *fctx;
28
29 fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
30 if (!fctx)
31 return ERR_PTR(-ENOMEM);
32
33 fctx->dev = dev;
34 strncpy(fctx->name, name, sizeof(fctx->name));
35 fctx->context = dma_fence_context_alloc(1);
36 init_waitqueue_head(&fctx->event);
37 spin_lock_init(&fctx->spinlock);
38
39 return fctx;
40}
41
42void msm_fence_context_free(struct msm_fence_context *fctx)
43{
44 kfree(fctx);
45}
46
47static inline bool fence_completed(struct msm_fence_context *fctx, uint32_t fence)
48{
49 return (int32_t)(fctx->completed_fence - fence) >= 0;
50}
51
52/* legacy path for WAIT_FENCE ioctl: */
53int msm_wait_fence(struct msm_fence_context *fctx, uint32_t fence,
54 ktime_t *timeout, bool interruptible)
55{
56 int ret;
57
58 if (fence > fctx->last_fence) {
59 DRM_ERROR("%s: waiting on invalid fence: %u (of %u)\n",
60 fctx->name, fence, fctx->last_fence);
61 return -EINVAL;
62 }
63
64 if (!timeout) {
65 /* no-wait: */
66 ret = fence_completed(fctx, fence) ? 0 : -EBUSY;
67 } else {
68 unsigned long remaining_jiffies = timeout_to_jiffies(timeout);
69
70 if (interruptible)
71 ret = wait_event_interruptible_timeout(fctx->event,
72 fence_completed(fctx, fence),
73 remaining_jiffies);
74 else
75 ret = wait_event_timeout(fctx->event,
76 fence_completed(fctx, fence),
77 remaining_jiffies);
78
79 if (ret == 0) {
80 DBG("timeout waiting for fence: %u (completed: %u)",
81 fence, fctx->completed_fence);
82 ret = -ETIMEDOUT;
83 } else if (ret != -ERESTARTSYS) {
84 ret = 0;
85 }
86 }
87
88 return ret;
89}
90
91/* called from workqueue */
92void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence)
93{
94 spin_lock(&fctx->spinlock);
95 fctx->completed_fence = max(fence, fctx->completed_fence);
96 spin_unlock(&fctx->spinlock);
97
98 wake_up_all(&fctx->event);
99}
100
101struct msm_fence {
102 struct dma_fence base;
103 struct msm_fence_context *fctx;
104};
105
106static inline struct msm_fence *to_msm_fence(struct dma_fence *fence)
107{
108 return container_of(fence, struct msm_fence, base);
109}
110
111static const char *msm_fence_get_driver_name(struct dma_fence *fence)
112{
113 return "msm";
114}
115
116static const char *msm_fence_get_timeline_name(struct dma_fence *fence)
117{
118 struct msm_fence *f = to_msm_fence(fence);
119 return f->fctx->name;
120}
121
122static bool msm_fence_enable_signaling(struct dma_fence *fence)
123{
124 return true;
125}
126
127static bool msm_fence_signaled(struct dma_fence *fence)
128{
129 struct msm_fence *f = to_msm_fence(fence);
130 return fence_completed(f->fctx, f->base.seqno);
131}
132
133static const struct dma_fence_ops msm_fence_ops = {
134 .get_driver_name = msm_fence_get_driver_name,
135 .get_timeline_name = msm_fence_get_timeline_name,
136 .enable_signaling = msm_fence_enable_signaling,
137 .signaled = msm_fence_signaled,
138 .wait = dma_fence_default_wait,
139 .release = dma_fence_free,
140};
141
142struct dma_fence *
143msm_fence_alloc(struct msm_fence_context *fctx)
144{
145 struct msm_fence *f;
146
147 f = kzalloc(sizeof(*f), GFP_KERNEL);
148 if (!f)
149 return ERR_PTR(-ENOMEM);
150
151 f->fctx = fctx;
152
153 dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
154 fctx->context, ++fctx->last_fence);
155
156 return &f->base;
157}