1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2019 Intel Corporation
  5 */
  6
  7#include <linux/slab.h>
  8#include <linux/workqueue.h>
  9
 10#include "i915_active.h"
 11#include "gem/i915_gem_context.h"
 12#include "gem/i915_gem_object.h"
 13#include "i915_globals.h"
 14#include "i915_request.h"
 15#include "i915_scheduler.h"
 16#include "i915_vma.h"
 17
 18static LIST_HEAD(globals);
 19
 20static atomic_t active;
 21static atomic_t epoch;
 22static struct park_work {
 23	struct rcu_work work;
 24	int epoch;
 25} park;
 26
 27static void i915_globals_shrink(void)
 28{
 29	struct i915_global *global;
 30
 31	/*
 32	 * kmem_cache_shrink() discards empty slabs and reorders partially
 33	 * filled slabs to prioritise allocating from the mostly full slabs,
 34	 * with the aim of reducing fragmentation.
 35	 */
 36	list_for_each_entry(global, &globals, link)
 37		global->shrink();
 38}
 39
 40static void __i915_globals_park(struct work_struct *work)
 41{
 42	/* Confirm nothing woke up in the last grace period */
 43	if (park.epoch == atomic_read(&epoch))
 44		i915_globals_shrink();
 45}
 46
 47void __init i915_global_register(struct i915_global *global)
 48{
 49	GEM_BUG_ON(!global->shrink);
 50	GEM_BUG_ON(!global->exit);
 51
 52	list_add_tail(&global->link, &globals);
 53}
 54
 55static void __i915_globals_cleanup(void)
 56{
 57	struct i915_global *global, *next;
 58
 59	list_for_each_entry_safe_reverse(global, next, &globals, link)
 60		global->exit();
 61}
 62
 63static __initconst int (* const initfn[])(void) = {
 64	i915_global_active_init,
 65	i915_global_buddy_init,
 66	i915_global_context_init,
 67	i915_global_gem_context_init,
 68	i915_global_objects_init,
 69	i915_global_request_init,
 70	i915_global_scheduler_init,
 71	i915_global_vma_init,
 72};
 73
 74int __init i915_globals_init(void)
 75{
 76	int i;
 77
 78	for (i = 0; i < ARRAY_SIZE(initfn); i++) {
 79		int err;
 80
 81		err = initfn[i]();
 82		if (err) {
 83			__i915_globals_cleanup();
 84			return err;
 85		}
 86	}
 87
 88	INIT_RCU_WORK(&park.work, __i915_globals_park);
 89	return 0;
 90}
 91
 92void i915_globals_park(void)
 93{
 94	/*
 95	 * Defer shrinking the global slab caches (and other work) until
 96	 * after a RCU grace period has completed with no activity. This
 97	 * is to try and reduce the latency impact on the consumers caused
 98	 * by us shrinking the caches the same time as they are trying to
 99	 * allocate, with the assumption being that if we idle long enough
100	 * for an RCU grace period to elapse since the last use, it is likely
101	 * to be longer until we need the caches again.
102	 */
103	if (!atomic_dec_and_test(&active))
104		return;
105
106	park.epoch = atomic_inc_return(&epoch);
107	queue_rcu_work(system_wq, &park.work);
108}
109
110void i915_globals_unpark(void)
111{
112	atomic_inc(&epoch);
113	atomic_inc(&active);
114}
115
116void __exit i915_globals_exit(void)
117{
118	/* Flush any residual park_work */
119	atomic_inc(&epoch);
120	flush_rcu_work(&park.work);
121
122	__i915_globals_cleanup();
123
124	/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
125	rcu_barrier();
126}