Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_pm.h"
7
8#include <linux/pm_runtime.h>
9
10#include <drm/drm_managed.h>
11#include <drm/ttm/ttm_placement.h>
12
13#include "xe_bo.h"
14#include "xe_bo_evict.h"
15#include "xe_device.h"
16#include "xe_device_sysfs.h"
17#include "xe_display.h"
18#include "xe_ggtt.h"
19#include "xe_gt.h"
20#include "xe_guc.h"
21#include "xe_irq.h"
22#include "xe_pcode.h"
23#include "xe_wa.h"
24
25/**
26 * DOC: Xe Power Management
27 *
28 * Xe PM shall be guided by the simplicity.
29 * Use the simplest hook options whenever possible.
30 * Let's not reinvent the runtime_pm references and hooks.
31 * Shall have a clear separation of display and gt underneath this component.
32 *
33 * What's next:
34 *
35 * For now s2idle and s3 are only working in integrated devices. The next step
36 * is to iterate through all VRAM's BO backing them up into the system memory
37 * before allowing the system suspend.
38 *
39 * Also runtime_pm needs to be here from the beginning.
40 *
41 * RC6/RPS are also critical PM features. Let's start with GuCRC and GuC SLPC
42 * and no wait boost. Frequency optimizations should come on a next stage.
43 */
44
45/**
46 * xe_pm_suspend - Helper for System suspend, i.e. S0->S3 / S0->S2idle
47 * @xe: xe device instance
48 *
49 * Return: 0 on success
50 */
51int xe_pm_suspend(struct xe_device *xe)
52{
53 struct xe_gt *gt;
54 u8 id;
55 int err;
56
57 for_each_gt(gt, xe, id)
58 xe_gt_suspend_prepare(gt);
59
60 /* FIXME: Super racey... */
61 err = xe_bo_evict_all(xe);
62 if (err)
63 return err;
64
65 xe_display_pm_suspend(xe);
66
67 for_each_gt(gt, xe, id) {
68 err = xe_gt_suspend(gt);
69 if (err) {
70 xe_display_pm_resume(xe);
71 return err;
72 }
73 }
74
75 xe_irq_suspend(xe);
76
77 xe_display_pm_suspend_late(xe);
78
79 return 0;
80}
81
82/**
83 * xe_pm_resume - Helper for System resume S3->S0 / S2idle->S0
84 * @xe: xe device instance
85 *
86 * Return: 0 on success
87 */
88int xe_pm_resume(struct xe_device *xe)
89{
90 struct xe_tile *tile;
91 struct xe_gt *gt;
92 u8 id;
93 int err;
94
95 for_each_tile(tile, xe, id)
96 xe_wa_apply_tile_workarounds(tile);
97
98 for_each_gt(gt, xe, id) {
99 err = xe_pcode_init(gt);
100 if (err)
101 return err;
102 }
103
104 xe_display_pm_resume_early(xe);
105
106 /*
107 * This only restores pinned memory which is the memory required for the
108 * GT(s) to resume.
109 */
110 err = xe_bo_restore_kernel(xe);
111 if (err)
112 return err;
113
114 xe_irq_resume(xe);
115
116 xe_display_pm_resume(xe);
117
118 for_each_gt(gt, xe, id)
119 xe_gt_resume(gt);
120
121 err = xe_bo_restore_user(xe);
122 if (err)
123 return err;
124
125 return 0;
126}
127
128static bool xe_pm_pci_d3cold_capable(struct pci_dev *pdev)
129{
130 struct pci_dev *root_pdev;
131
132 root_pdev = pcie_find_root_port(pdev);
133 if (!root_pdev)
134 return false;
135
136 /* D3Cold requires PME capability and _PR3 power resource */
137 if (!pci_pme_capable(root_pdev, PCI_D3cold) || !pci_pr3_present(root_pdev))
138 return false;
139
140 return true;
141}
142
143static void xe_pm_runtime_init(struct xe_device *xe)
144{
145 struct device *dev = xe->drm.dev;
146
147 /*
148 * Disable the system suspend direct complete optimization.
149 * We need to ensure that the regular device suspend/resume functions
150 * are called since our runtime_pm cannot guarantee local memory
151 * eviction for d3cold.
152 * TODO: Check HDA audio dependencies claimed by i915, and then enforce
153 * this option to integrated graphics as well.
154 */
155 if (IS_DGFX(xe))
156 dev_pm_set_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);
157
158 pm_runtime_use_autosuspend(dev);
159 pm_runtime_set_autosuspend_delay(dev, 1000);
160 pm_runtime_set_active(dev);
161 pm_runtime_allow(dev);
162 pm_runtime_mark_last_busy(dev);
163 pm_runtime_put(dev);
164}
165
166void xe_pm_init(struct xe_device *xe)
167{
168 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
169
170 /* For now suspend/resume is only allowed with GuC */
171 if (!xe_device_uc_enabled(xe))
172 return;
173
174 drmm_mutex_init(&xe->drm, &xe->d3cold.lock);
175
176 xe->d3cold.capable = xe_pm_pci_d3cold_capable(pdev);
177
178 if (xe->d3cold.capable) {
179 xe_device_sysfs_init(xe);
180 xe_pm_set_vram_threshold(xe, DEFAULT_VRAM_THRESHOLD);
181 }
182
183 xe_pm_runtime_init(xe);
184}
185
186void xe_pm_runtime_fini(struct xe_device *xe)
187{
188 struct device *dev = xe->drm.dev;
189
190 pm_runtime_get_sync(dev);
191 pm_runtime_forbid(dev);
192}
193
194static void xe_pm_write_callback_task(struct xe_device *xe,
195 struct task_struct *task)
196{
197 WRITE_ONCE(xe->pm_callback_task, task);
198
199 /*
200 * Just in case it's somehow possible for our writes to be reordered to
201 * the extent that something else re-uses the task written in
202 * pm_callback_task. For example after returning from the callback, but
203 * before the reordered write that resets pm_callback_task back to NULL.
204 */
205 smp_mb(); /* pairs with xe_pm_read_callback_task */
206}
207
208struct task_struct *xe_pm_read_callback_task(struct xe_device *xe)
209{
210 smp_mb(); /* pairs with xe_pm_write_callback_task */
211
212 return READ_ONCE(xe->pm_callback_task);
213}
214
215int xe_pm_runtime_suspend(struct xe_device *xe)
216{
217 struct xe_gt *gt;
218 u8 id;
219 int err = 0;
220
221 if (xe->d3cold.allowed && xe_device_mem_access_ongoing(xe))
222 return -EBUSY;
223
224 /* Disable access_ongoing asserts and prevent recursive pm calls */
225 xe_pm_write_callback_task(xe, current);
226
227 /*
228 * The actual xe_device_mem_access_put() is always async underneath, so
229 * exactly where that is called should makes no difference to us. However
230 * we still need to be very careful with the locks that this callback
231 * acquires and the locks that are acquired and held by any callers of
232 * xe_device_mem_access_get(). We already have the matching annotation
233 * on that side, but we also need it here. For example lockdep should be
234 * able to tell us if the following scenario is in theory possible:
235 *
236 * CPU0 | CPU1 (kworker)
237 * lock(A) |
238 * | xe_pm_runtime_suspend()
239 * | lock(A)
240 * xe_device_mem_access_get() |
241 *
242 * This will clearly deadlock since rpm core needs to wait for
243 * xe_pm_runtime_suspend() to complete, but here we are holding lock(A)
244 * on CPU0 which prevents CPU1 making forward progress. With the
245 * annotation here and in xe_device_mem_access_get() lockdep will see
246 * the potential lock inversion and give us a nice splat.
247 */
248 lock_map_acquire(&xe_device_mem_access_lockdep_map);
249
250 if (xe->d3cold.allowed) {
251 err = xe_bo_evict_all(xe);
252 if (err)
253 goto out;
254 }
255
256 for_each_gt(gt, xe, id) {
257 err = xe_gt_suspend(gt);
258 if (err)
259 goto out;
260 }
261
262 xe_irq_suspend(xe);
263out:
264 lock_map_release(&xe_device_mem_access_lockdep_map);
265 xe_pm_write_callback_task(xe, NULL);
266 return err;
267}
268
269int xe_pm_runtime_resume(struct xe_device *xe)
270{
271 struct xe_gt *gt;
272 u8 id;
273 int err = 0;
274
275 /* Disable access_ongoing asserts and prevent recursive pm calls */
276 xe_pm_write_callback_task(xe, current);
277
278 lock_map_acquire(&xe_device_mem_access_lockdep_map);
279
280 /*
281 * It can be possible that xe has allowed d3cold but other pcie devices
282 * in gfx card soc would have blocked d3cold, therefore card has not
283 * really lost power. Detecting primary Gt power is sufficient.
284 */
285 gt = xe_device_get_gt(xe, 0);
286 xe->d3cold.power_lost = xe_guc_in_reset(>->uc.guc);
287
288 if (xe->d3cold.allowed && xe->d3cold.power_lost) {
289 for_each_gt(gt, xe, id) {
290 err = xe_pcode_init(gt);
291 if (err)
292 goto out;
293 }
294
295 /*
296 * This only restores pinned memory which is the memory
297 * required for the GT(s) to resume.
298 */
299 err = xe_bo_restore_kernel(xe);
300 if (err)
301 goto out;
302 }
303
304 xe_irq_resume(xe);
305
306 for_each_gt(gt, xe, id)
307 xe_gt_resume(gt);
308
309 if (xe->d3cold.allowed && xe->d3cold.power_lost) {
310 err = xe_bo_restore_user(xe);
311 if (err)
312 goto out;
313 }
314out:
315 lock_map_release(&xe_device_mem_access_lockdep_map);
316 xe_pm_write_callback_task(xe, NULL);
317 return err;
318}
319
320int xe_pm_runtime_get(struct xe_device *xe)
321{
322 return pm_runtime_get_sync(xe->drm.dev);
323}
324
325int xe_pm_runtime_put(struct xe_device *xe)
326{
327 pm_runtime_mark_last_busy(xe->drm.dev);
328 return pm_runtime_put(xe->drm.dev);
329}
330
331int xe_pm_runtime_get_if_active(struct xe_device *xe)
332{
333 return pm_runtime_get_if_active(xe->drm.dev, true);
334}
335
336void xe_pm_assert_unbounded_bridge(struct xe_device *xe)
337{
338 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
339 struct pci_dev *bridge = pci_upstream_bridge(pdev);
340
341 if (!bridge)
342 return;
343
344 if (!bridge->driver) {
345 drm_warn(&xe->drm, "unbounded parent pci bridge, device won't support any PM support.\n");
346 device_set_pm_not_required(&pdev->dev);
347 }
348}
349
350int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold)
351{
352 struct ttm_resource_manager *man;
353 u32 vram_total_mb = 0;
354 int i;
355
356 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
357 man = ttm_manager_type(&xe->ttm, i);
358 if (man)
359 vram_total_mb += DIV_ROUND_UP_ULL(man->size, 1024 * 1024);
360 }
361
362 drm_dbg(&xe->drm, "Total vram %u mb\n", vram_total_mb);
363
364 if (threshold > vram_total_mb)
365 return -EINVAL;
366
367 mutex_lock(&xe->d3cold.lock);
368 xe->d3cold.vram_threshold = threshold;
369 mutex_unlock(&xe->d3cold.lock);
370
371 return 0;
372}
373
374void xe_pm_d3cold_allowed_toggle(struct xe_device *xe)
375{
376 struct ttm_resource_manager *man;
377 u32 total_vram_used_mb = 0;
378 u64 vram_used;
379 int i;
380
381 if (!xe->d3cold.capable) {
382 xe->d3cold.allowed = false;
383 return;
384 }
385
386 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
387 man = ttm_manager_type(&xe->ttm, i);
388 if (man) {
389 vram_used = ttm_resource_manager_usage(man);
390 total_vram_used_mb += DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
391 }
392 }
393
394 mutex_lock(&xe->d3cold.lock);
395
396 if (total_vram_used_mb < xe->d3cold.vram_threshold)
397 xe->d3cold.allowed = true;
398 else
399 xe->d3cold.allowed = false;
400
401 mutex_unlock(&xe->d3cold.lock);
402
403 drm_dbg(&xe->drm,
404 "d3cold: allowed=%s\n", str_yes_no(xe->d3cold.allowed));
405}