Loading...
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7#include <linux/wait_bit.h>
8
9#include "intel_runtime_pm.h"
10#include "intel_wakeref.h"
11
12static void rpm_get(struct intel_wakeref *wf)
13{
14 wf->wakeref = intel_runtime_pm_get(wf->rpm);
15}
16
17static void rpm_put(struct intel_wakeref *wf)
18{
19 intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref);
20
21 intel_runtime_pm_put(wf->rpm, wakeref);
22 INTEL_WAKEREF_BUG_ON(!wakeref);
23}
24
25int __intel_wakeref_get_first(struct intel_wakeref *wf)
26{
27 /*
28 * Treat get/put as different subclasses, as we may need to run
29 * the put callback from under the shrinker and do not want to
30 * cross-contanimate that callback with any extra work performed
31 * upon acquiring the wakeref.
32 */
33 mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
34 if (!atomic_read(&wf->count)) {
35 int err;
36
37 rpm_get(wf);
38
39 err = wf->ops->get(wf);
40 if (unlikely(err)) {
41 rpm_put(wf);
42 mutex_unlock(&wf->mutex);
43 return err;
44 }
45
46 smp_mb__before_atomic(); /* release wf->count */
47 }
48 atomic_inc(&wf->count);
49 mutex_unlock(&wf->mutex);
50
51 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
52 return 0;
53}
54
55static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
56{
57 if (!atomic_dec_and_test(&wf->count))
58 goto unlock;
59
60 /* ops->put() must reschedule its own release on error/deferral */
61 if (likely(!wf->ops->put(wf))) {
62 rpm_put(wf);
63 wake_up_var(&wf->wakeref);
64 }
65
66unlock:
67 mutex_unlock(&wf->mutex);
68}
69
70void __intel_wakeref_put_last(struct intel_wakeref *wf)
71{
72 INTEL_WAKEREF_BUG_ON(work_pending(&wf->work));
73
74 /* Assume we are not in process context and so cannot sleep. */
75 if (wf->ops->flags & INTEL_WAKEREF_PUT_ASYNC ||
76 !mutex_trylock(&wf->mutex)) {
77 schedule_work(&wf->work);
78 return;
79 }
80
81 ____intel_wakeref_put_last(wf);
82}
83
84static void __intel_wakeref_put_work(struct work_struct *wrk)
85{
86 struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work);
87
88 if (atomic_add_unless(&wf->count, -1, 1))
89 return;
90
91 mutex_lock(&wf->mutex);
92 ____intel_wakeref_put_last(wf);
93}
94
95void __intel_wakeref_init(struct intel_wakeref *wf,
96 struct intel_runtime_pm *rpm,
97 const struct intel_wakeref_ops *ops,
98 struct lock_class_key *key)
99{
100 wf->rpm = rpm;
101 wf->ops = ops;
102
103 __mutex_init(&wf->mutex, "wakeref", key);
104 atomic_set(&wf->count, 0);
105 wf->wakeref = 0;
106
107 INIT_WORK(&wf->work, __intel_wakeref_put_work);
108}
109
110int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
111{
112 return wait_var_event_killable(&wf->wakeref,
113 !intel_wakeref_is_active(wf));
114}
115
116static void wakeref_auto_timeout(struct timer_list *t)
117{
118 struct intel_wakeref_auto *wf = from_timer(wf, t, timer);
119 intel_wakeref_t wakeref;
120 unsigned long flags;
121
122 if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))
123 return;
124
125 wakeref = fetch_and_zero(&wf->wakeref);
126 spin_unlock_irqrestore(&wf->lock, flags);
127
128 intel_runtime_pm_put(wf->rpm, wakeref);
129}
130
131void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
132 struct intel_runtime_pm *rpm)
133{
134 spin_lock_init(&wf->lock);
135 timer_setup(&wf->timer, wakeref_auto_timeout, 0);
136 refcount_set(&wf->count, 0);
137 wf->wakeref = 0;
138 wf->rpm = rpm;
139}
140
141void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
142{
143 unsigned long flags;
144
145 if (!timeout) {
146 if (del_timer_sync(&wf->timer))
147 wakeref_auto_timeout(&wf->timer);
148 return;
149 }
150
151 /* Our mission is that we only extend an already active wakeref */
152 assert_rpm_wakelock_held(wf->rpm);
153
154 if (!refcount_inc_not_zero(&wf->count)) {
155 spin_lock_irqsave(&wf->lock, flags);
156 if (!refcount_inc_not_zero(&wf->count)) {
157 INTEL_WAKEREF_BUG_ON(wf->wakeref);
158 wf->wakeref = intel_runtime_pm_get_if_in_use(wf->rpm);
159 refcount_set(&wf->count, 1);
160 }
161 spin_unlock_irqrestore(&wf->lock, flags);
162 }
163
164 /*
165 * If we extend a pending timer, we will only get a single timer
166 * callback and so need to cancel the local inc by running the
167 * elided callback to keep the wf->count balanced.
168 */
169 if (mod_timer(&wf->timer, jiffies + timeout))
170 wakeref_auto_timeout(&wf->timer);
171}
172
173void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
174{
175 intel_wakeref_auto(wf, 0);
176 INTEL_WAKEREF_BUG_ON(wf->wakeref);
177}
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7#include <linux/wait_bit.h>
8
9#include "intel_runtime_pm.h"
10#include "intel_wakeref.h"
11#include "i915_drv.h"
12
13int __intel_wakeref_get_first(struct intel_wakeref *wf)
14{
15 intel_wakeref_t wakeref;
16 int ret = 0;
17
18 wakeref = intel_runtime_pm_get(&wf->i915->runtime_pm);
19 /*
20 * Treat get/put as different subclasses, as we may need to run
21 * the put callback from under the shrinker and do not want to
22 * cross-contanimate that callback with any extra work performed
23 * upon acquiring the wakeref.
24 */
25 mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
26
27 if (!atomic_read(&wf->count)) {
28 INTEL_WAKEREF_BUG_ON(wf->wakeref);
29 wf->wakeref = wakeref;
30 wakeref = 0;
31
32 ret = wf->ops->get(wf);
33 if (ret) {
34 wakeref = xchg(&wf->wakeref, 0);
35 wake_up_var(&wf->wakeref);
36 goto unlock;
37 }
38
39 smp_mb__before_atomic(); /* release wf->count */
40 }
41
42 atomic_inc(&wf->count);
43 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
44
45unlock:
46 mutex_unlock(&wf->mutex);
47 if (unlikely(wakeref))
48 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
49
50 return ret;
51}
52
53static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
54{
55 intel_wakeref_t wakeref = 0;
56
57 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
58 if (unlikely(!atomic_dec_and_test(&wf->count)))
59 goto unlock;
60
61 /* ops->put() must reschedule its own release on error/deferral */
62 if (likely(!wf->ops->put(wf))) {
63 INTEL_WAKEREF_BUG_ON(!wf->wakeref);
64 wakeref = xchg(&wf->wakeref, 0);
65 wake_up_var(&wf->wakeref);
66 }
67
68unlock:
69 mutex_unlock(&wf->mutex);
70 if (wakeref)
71 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
72}
73
74void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
75{
76 INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work));
77
78 /* Assume we are not in process context and so cannot sleep. */
79 if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
80 mod_delayed_work(wf->i915->unordered_wq, &wf->work,
81 FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags));
82 return;
83 }
84
85 ____intel_wakeref_put_last(wf);
86}
87
88static void __intel_wakeref_put_work(struct work_struct *wrk)
89{
90 struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work);
91
92 if (atomic_add_unless(&wf->count, -1, 1))
93 return;
94
95 mutex_lock(&wf->mutex);
96 ____intel_wakeref_put_last(wf);
97}
98
99void __intel_wakeref_init(struct intel_wakeref *wf,
100 struct drm_i915_private *i915,
101 const struct intel_wakeref_ops *ops,
102 struct intel_wakeref_lockclass *key,
103 const char *name)
104{
105 wf->i915 = i915;
106 wf->ops = ops;
107
108 __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex);
109 atomic_set(&wf->count, 0);
110 wf->wakeref = 0;
111
112 INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work);
113 lockdep_init_map(&wf->work.work.lockdep_map,
114 "wakeref.work", &key->work, 0);
115
116#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_WAKEREF)
117 ref_tracker_dir_init(&wf->debug, INTEL_REFTRACK_DEAD_COUNT, name);
118#endif
119}
120
121int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
122{
123 int err;
124
125 might_sleep();
126
127 err = wait_var_event_killable(&wf->wakeref,
128 !intel_wakeref_is_active(wf));
129 if (err)
130 return err;
131
132 intel_wakeref_unlock_wait(wf);
133 return 0;
134}
135
136static void wakeref_auto_timeout(struct timer_list *t)
137{
138 struct intel_wakeref_auto *wf = from_timer(wf, t, timer);
139 intel_wakeref_t wakeref;
140 unsigned long flags;
141
142 if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))
143 return;
144
145 wakeref = fetch_and_zero(&wf->wakeref);
146 spin_unlock_irqrestore(&wf->lock, flags);
147
148 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
149}
150
151void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
152 struct drm_i915_private *i915)
153{
154 spin_lock_init(&wf->lock);
155 timer_setup(&wf->timer, wakeref_auto_timeout, 0);
156 refcount_set(&wf->count, 0);
157 wf->wakeref = 0;
158 wf->i915 = i915;
159}
160
161void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
162{
163 unsigned long flags;
164
165 if (!timeout) {
166 if (del_timer_sync(&wf->timer))
167 wakeref_auto_timeout(&wf->timer);
168 return;
169 }
170
171 /* Our mission is that we only extend an already active wakeref */
172 assert_rpm_wakelock_held(&wf->i915->runtime_pm);
173
174 if (!refcount_inc_not_zero(&wf->count)) {
175 spin_lock_irqsave(&wf->lock, flags);
176 if (!refcount_inc_not_zero(&wf->count)) {
177 INTEL_WAKEREF_BUG_ON(wf->wakeref);
178 wf->wakeref =
179 intel_runtime_pm_get_if_in_use(&wf->i915->runtime_pm);
180 refcount_set(&wf->count, 1);
181 }
182 spin_unlock_irqrestore(&wf->lock, flags);
183 }
184
185 /*
186 * If we extend a pending timer, we will only get a single timer
187 * callback and so need to cancel the local inc by running the
188 * elided callback to keep the wf->count balanced.
189 */
190 if (mod_timer(&wf->timer, jiffies + timeout))
191 wakeref_auto_timeout(&wf->timer);
192}
193
194void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
195{
196 intel_wakeref_auto(wf, 0);
197 INTEL_WAKEREF_BUG_ON(wf->wakeref);
198}
199
200void intel_ref_tracker_show(struct ref_tracker_dir *dir,
201 struct drm_printer *p)
202{
203 const size_t buf_size = PAGE_SIZE;
204 char *buf, *sb, *se;
205 size_t count;
206
207 buf = kmalloc(buf_size, GFP_NOWAIT);
208 if (!buf)
209 return;
210
211 count = ref_tracker_dir_snprint(dir, buf, buf_size);
212 if (!count)
213 goto free;
214 /* printk does not like big buffers, so we split it */
215 for (sb = buf; *sb; sb = se + 1) {
216 se = strchrnul(sb, '\n');
217 drm_printf(p, "%.*s", (int)(se - sb + 1), sb);
218 if (!*se)
219 break;
220 }
221 if (count >= buf_size)
222 drm_printf(p, "\n...dropped %zd extra bytes of leak report.\n",
223 count + 1 - buf_size);
224free:
225 kfree(buf);
226}