Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_pm.h"
7
8#include <linux/pm_runtime.h>
9
10#include <drm/drm_managed.h>
11#include <drm/ttm/ttm_placement.h>
12
13#include "display/xe_display.h"
14#include "xe_bo.h"
15#include "xe_bo_evict.h"
16#include "xe_device.h"
17#include "xe_device_sysfs.h"
18#include "xe_ggtt.h"
19#include "xe_gt.h"
20#include "xe_guc.h"
21#include "xe_irq.h"
22#include "xe_pcode.h"
23#include "xe_wa.h"
24
25/**
26 * DOC: Xe Power Management
27 *
28 * Xe PM shall be guided by the simplicity.
29 * Use the simplest hook options whenever possible.
30 * Let's not reinvent the runtime_pm references and hooks.
31 * Shall have a clear separation of display and gt underneath this component.
32 *
33 * What's next:
34 *
35 * For now s2idle and s3 are only working in integrated devices. The next step
36 * is to iterate through all VRAM's BO backing them up into the system memory
37 * before allowing the system suspend.
38 *
39 * Also runtime_pm needs to be here from the beginning.
40 *
41 * RC6/RPS are also critical PM features. Let's start with GuCRC and GuC SLPC
42 * and no wait boost. Frequency optimizations should come on a next stage.
43 */
44
45/**
46 * xe_pm_suspend - Helper for System suspend, i.e. S0->S3 / S0->S2idle
47 * @xe: xe device instance
48 *
49 * Return: 0 on success
50 */
51int xe_pm_suspend(struct xe_device *xe)
52{
53 struct xe_gt *gt;
54 u8 id;
55 int err;
56
57 drm_dbg(&xe->drm, "Suspending device\n");
58
59 for_each_gt(gt, xe, id)
60 xe_gt_suspend_prepare(gt);
61
62 /* FIXME: Super racey... */
63 err = xe_bo_evict_all(xe);
64 if (err)
65 goto err;
66
67 xe_display_pm_suspend(xe);
68
69 for_each_gt(gt, xe, id) {
70 err = xe_gt_suspend(gt);
71 if (err) {
72 xe_display_pm_resume(xe);
73 goto err;
74 }
75 }
76
77 xe_irq_suspend(xe);
78
79 xe_display_pm_suspend_late(xe);
80
81 drm_dbg(&xe->drm, "Device suspended\n");
82 return 0;
83err:
84 drm_dbg(&xe->drm, "Device suspend failed %d\n", err);
85 return err;
86}
87
88/**
89 * xe_pm_resume - Helper for System resume S3->S0 / S2idle->S0
90 * @xe: xe device instance
91 *
92 * Return: 0 on success
93 */
94int xe_pm_resume(struct xe_device *xe)
95{
96 struct xe_tile *tile;
97 struct xe_gt *gt;
98 u8 id;
99 int err;
100
101 drm_dbg(&xe->drm, "Resuming device\n");
102
103 for_each_tile(tile, xe, id)
104 xe_wa_apply_tile_workarounds(tile);
105
106 err = xe_pcode_ready(xe, true);
107 if (err)
108 return err;
109
110 xe_display_pm_resume_early(xe);
111
112 /*
113 * This only restores pinned memory which is the memory required for the
114 * GT(s) to resume.
115 */
116 err = xe_bo_restore_kernel(xe);
117 if (err)
118 goto err;
119
120 xe_irq_resume(xe);
121
122 xe_display_pm_resume(xe);
123
124 for_each_gt(gt, xe, id)
125 xe_gt_resume(gt);
126
127 err = xe_bo_restore_user(xe);
128 if (err)
129 goto err;
130
131 drm_dbg(&xe->drm, "Device resumed\n");
132 return 0;
133err:
134 drm_dbg(&xe->drm, "Device resume failed %d\n", err);
135 return err;
136}
137
138static bool xe_pm_pci_d3cold_capable(struct xe_device *xe)
139{
140 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
141 struct pci_dev *root_pdev;
142
143 root_pdev = pcie_find_root_port(pdev);
144 if (!root_pdev)
145 return false;
146
147 /* D3Cold requires PME capability */
148 if (!pci_pme_capable(root_pdev, PCI_D3cold)) {
149 drm_dbg(&xe->drm, "d3cold: PME# not supported\n");
150 return false;
151 }
152
153 /* D3Cold requires _PR3 power resource */
154 if (!pci_pr3_present(root_pdev)) {
155 drm_dbg(&xe->drm, "d3cold: ACPI _PR3 not present\n");
156 return false;
157 }
158
159 return true;
160}
161
162static void xe_pm_runtime_init(struct xe_device *xe)
163{
164 struct device *dev = xe->drm.dev;
165
166 /*
167 * Disable the system suspend direct complete optimization.
168 * We need to ensure that the regular device suspend/resume functions
169 * are called since our runtime_pm cannot guarantee local memory
170 * eviction for d3cold.
171 * TODO: Check HDA audio dependencies claimed by i915, and then enforce
172 * this option to integrated graphics as well.
173 */
174 if (IS_DGFX(xe))
175 dev_pm_set_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);
176
177 pm_runtime_use_autosuspend(dev);
178 pm_runtime_set_autosuspend_delay(dev, 1000);
179 pm_runtime_set_active(dev);
180 pm_runtime_allow(dev);
181 pm_runtime_mark_last_busy(dev);
182 pm_runtime_put(dev);
183}
184
185void xe_pm_init_early(struct xe_device *xe)
186{
187 INIT_LIST_HEAD(&xe->mem_access.vram_userfault.list);
188 drmm_mutex_init(&xe->drm, &xe->mem_access.vram_userfault.lock);
189}
190
191void xe_pm_init(struct xe_device *xe)
192{
193 /* For now suspend/resume is only allowed with GuC */
194 if (!xe_device_uc_enabled(xe))
195 return;
196
197 drmm_mutex_init(&xe->drm, &xe->d3cold.lock);
198
199 xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe);
200
201 if (xe->d3cold.capable) {
202 xe_device_sysfs_init(xe);
203 xe_pm_set_vram_threshold(xe, DEFAULT_VRAM_THRESHOLD);
204 }
205
206 xe_pm_runtime_init(xe);
207}
208
209void xe_pm_runtime_fini(struct xe_device *xe)
210{
211 struct device *dev = xe->drm.dev;
212
213 pm_runtime_get_sync(dev);
214 pm_runtime_forbid(dev);
215}
216
217static void xe_pm_write_callback_task(struct xe_device *xe,
218 struct task_struct *task)
219{
220 WRITE_ONCE(xe->pm_callback_task, task);
221
222 /*
223 * Just in case it's somehow possible for our writes to be reordered to
224 * the extent that something else re-uses the task written in
225 * pm_callback_task. For example after returning from the callback, but
226 * before the reordered write that resets pm_callback_task back to NULL.
227 */
228 smp_mb(); /* pairs with xe_pm_read_callback_task */
229}
230
231struct task_struct *xe_pm_read_callback_task(struct xe_device *xe)
232{
233 smp_mb(); /* pairs with xe_pm_write_callback_task */
234
235 return READ_ONCE(xe->pm_callback_task);
236}
237
238int xe_pm_runtime_suspend(struct xe_device *xe)
239{
240 struct xe_bo *bo, *on;
241 struct xe_gt *gt;
242 u8 id;
243 int err = 0;
244
245 if (xe->d3cold.allowed && xe_device_mem_access_ongoing(xe))
246 return -EBUSY;
247
248 /* Disable access_ongoing asserts and prevent recursive pm calls */
249 xe_pm_write_callback_task(xe, current);
250
251 /*
252 * The actual xe_device_mem_access_put() is always async underneath, so
253 * exactly where that is called should makes no difference to us. However
254 * we still need to be very careful with the locks that this callback
255 * acquires and the locks that are acquired and held by any callers of
256 * xe_device_mem_access_get(). We already have the matching annotation
257 * on that side, but we also need it here. For example lockdep should be
258 * able to tell us if the following scenario is in theory possible:
259 *
260 * CPU0 | CPU1 (kworker)
261 * lock(A) |
262 * | xe_pm_runtime_suspend()
263 * | lock(A)
264 * xe_device_mem_access_get() |
265 *
266 * This will clearly deadlock since rpm core needs to wait for
267 * xe_pm_runtime_suspend() to complete, but here we are holding lock(A)
268 * on CPU0 which prevents CPU1 making forward progress. With the
269 * annotation here and in xe_device_mem_access_get() lockdep will see
270 * the potential lock inversion and give us a nice splat.
271 */
272 lock_map_acquire(&xe_device_mem_access_lockdep_map);
273
274 /*
275 * Applying lock for entire list op as xe_ttm_bo_destroy and xe_bo_move_notify
276 * also checks and delets bo entry from user fault list.
277 */
278 mutex_lock(&xe->mem_access.vram_userfault.lock);
279 list_for_each_entry_safe(bo, on,
280 &xe->mem_access.vram_userfault.list, vram_userfault_link)
281 xe_bo_runtime_pm_release_mmap_offset(bo);
282 mutex_unlock(&xe->mem_access.vram_userfault.lock);
283
284 if (xe->d3cold.allowed) {
285 err = xe_bo_evict_all(xe);
286 if (err)
287 goto out;
288 }
289
290 for_each_gt(gt, xe, id) {
291 err = xe_gt_suspend(gt);
292 if (err)
293 goto out;
294 }
295
296 xe_irq_suspend(xe);
297out:
298 lock_map_release(&xe_device_mem_access_lockdep_map);
299 xe_pm_write_callback_task(xe, NULL);
300 return err;
301}
302
303int xe_pm_runtime_resume(struct xe_device *xe)
304{
305 struct xe_gt *gt;
306 u8 id;
307 int err = 0;
308
309 /* Disable access_ongoing asserts and prevent recursive pm calls */
310 xe_pm_write_callback_task(xe, current);
311
312 lock_map_acquire(&xe_device_mem_access_lockdep_map);
313
314 /*
315 * It can be possible that xe has allowed d3cold but other pcie devices
316 * in gfx card soc would have blocked d3cold, therefore card has not
317 * really lost power. Detecting primary Gt power is sufficient.
318 */
319 gt = xe_device_get_gt(xe, 0);
320 xe->d3cold.power_lost = xe_guc_in_reset(>->uc.guc);
321
322 if (xe->d3cold.allowed && xe->d3cold.power_lost) {
323 err = xe_pcode_ready(xe, true);
324 if (err)
325 goto out;
326
327 /*
328 * This only restores pinned memory which is the memory
329 * required for the GT(s) to resume.
330 */
331 err = xe_bo_restore_kernel(xe);
332 if (err)
333 goto out;
334 }
335
336 xe_irq_resume(xe);
337
338 for_each_gt(gt, xe, id)
339 xe_gt_resume(gt);
340
341 if (xe->d3cold.allowed && xe->d3cold.power_lost) {
342 err = xe_bo_restore_user(xe);
343 if (err)
344 goto out;
345 }
346out:
347 lock_map_release(&xe_device_mem_access_lockdep_map);
348 xe_pm_write_callback_task(xe, NULL);
349 return err;
350}
351
352int xe_pm_runtime_get(struct xe_device *xe)
353{
354 return pm_runtime_get_sync(xe->drm.dev);
355}
356
357int xe_pm_runtime_put(struct xe_device *xe)
358{
359 pm_runtime_mark_last_busy(xe->drm.dev);
360 return pm_runtime_put(xe->drm.dev);
361}
362
363int xe_pm_runtime_get_if_active(struct xe_device *xe)
364{
365 return pm_runtime_get_if_active(xe->drm.dev);
366}
367
368void xe_pm_assert_unbounded_bridge(struct xe_device *xe)
369{
370 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
371 struct pci_dev *bridge = pci_upstream_bridge(pdev);
372
373 if (!bridge)
374 return;
375
376 if (!bridge->driver) {
377 drm_warn(&xe->drm, "unbounded parent pci bridge, device won't support any PM support.\n");
378 device_set_pm_not_required(&pdev->dev);
379 }
380}
381
382int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold)
383{
384 struct ttm_resource_manager *man;
385 u32 vram_total_mb = 0;
386 int i;
387
388 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
389 man = ttm_manager_type(&xe->ttm, i);
390 if (man)
391 vram_total_mb += DIV_ROUND_UP_ULL(man->size, 1024 * 1024);
392 }
393
394 drm_dbg(&xe->drm, "Total vram %u mb\n", vram_total_mb);
395
396 if (threshold > vram_total_mb)
397 return -EINVAL;
398
399 mutex_lock(&xe->d3cold.lock);
400 xe->d3cold.vram_threshold = threshold;
401 mutex_unlock(&xe->d3cold.lock);
402
403 return 0;
404}
405
406void xe_pm_d3cold_allowed_toggle(struct xe_device *xe)
407{
408 struct ttm_resource_manager *man;
409 u32 total_vram_used_mb = 0;
410 u64 vram_used;
411 int i;
412
413 if (!xe->d3cold.capable) {
414 xe->d3cold.allowed = false;
415 return;
416 }
417
418 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
419 man = ttm_manager_type(&xe->ttm, i);
420 if (man) {
421 vram_used = ttm_resource_manager_usage(man);
422 total_vram_used_mb += DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
423 }
424 }
425
426 mutex_lock(&xe->d3cold.lock);
427
428 if (total_vram_used_mb < xe->d3cold.vram_threshold)
429 xe->d3cold.allowed = true;
430 else
431 xe->d3cold.allowed = false;
432
433 mutex_unlock(&xe->d3cold.lock);
434
435 drm_dbg(&xe->drm,
436 "d3cold: allowed=%s\n", str_yes_no(xe->d3cold.allowed));
437}