Linux Audio

Check our new training course

Loading...
Note: File does not exist in v6.13.7.
  1/*
  2 * Copyright (c) Red Hat Inc.
  3
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the
 12 * next paragraph) shall be included in all copies or substantial portions
 13 * of the Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 21 * DEALINGS IN THE SOFTWARE.
 22 *
 23 * Authors: Dave Airlie <airlied@redhat.com>
 24 *          Jerome Glisse <jglisse@redhat.com>
 25 *          Pauli Nieminen <suokkos@gmail.com>
 26 */
 27
 28/* simple list based uncached page pool
 29 * - Pool collects resently freed pages for reuse
 30 * - Use page->lru to keep a free list
 31 * - doesn't track currently in use pages
 32 */
 33#include <linux/list.h>
 34#include <linux/spinlock.h>
 35#include <linux/highmem.h>
 36#include <linux/mm_types.h>
 37#include <linux/module.h>
 38#include <linux/mm.h>
 39#include <linux/seq_file.h> /* for seq_printf */
 40#include <linux/slab.h>
 41#include <linux/dma-mapping.h>
 42
 43#include <linux/atomic.h>
 44
 45#include "ttm/ttm_bo_driver.h"
 46#include "ttm/ttm_page_alloc.h"
 47
 48#ifdef TTM_HAS_AGP
 49#include <asm/agp.h>
 50#endif
 51
 52#define NUM_PAGES_TO_ALLOC		(PAGE_SIZE/sizeof(struct page *))
 53#define SMALL_ALLOCATION		16
 54#define FREE_ALL_PAGES			(~0U)
 55/* times are in msecs */
 56#define PAGE_FREE_INTERVAL		1000
 57
 58/**
 59 * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
 60 *
 61 * @lock: Protects the shared pool from concurrnet access. Must be used with
 62 * irqsave/irqrestore variants because pool allocator maybe called from
 63 * delayed work.
 64 * @fill_lock: Prevent concurrent calls to fill.
 65 * @list: Pool of free uc/wc pages for fast reuse.
 66 * @gfp_flags: Flags to pass for alloc_page.
 67 * @npages: Number of pages in pool.
 68 */
 69struct ttm_page_pool {
 70	spinlock_t		lock;
 71	bool			fill_lock;
 72	struct list_head	list;
 73	gfp_t			gfp_flags;
 74	unsigned		npages;
 75	char			*name;
 76	unsigned long		nfrees;
 77	unsigned long		nrefills;
 78};
 79
 80/**
 81 * Limits for the pool. They are handled without locks because only place where
 82 * they may change is in sysfs store. They won't have immediate effect anyway
 83 * so forcing serialization to access them is pointless.
 84 */
 85
 86struct ttm_pool_opts {
 87	unsigned	alloc_size;
 88	unsigned	max_size;
 89	unsigned	small;
 90};
 91
 92#define NUM_POOLS 4
 93
 94/**
 95 * struct ttm_pool_manager - Holds memory pools for fst allocation
 96 *
 97 * Manager is read only object for pool code so it doesn't need locking.
 98 *
 99 * @free_interval: minimum number of jiffies between freeing pages from pool.
100 * @page_alloc_inited: reference counting for pool allocation.
101 * @work: Work that is used to shrink the pool. Work is only run when there is
102 * some pages to free.
103 * @small_allocation: Limit in number of pages what is small allocation.
104 *
105 * @pools: All pool objects in use.
106 **/
107struct ttm_pool_manager {
108	struct kobject		kobj;
109	struct shrinker		mm_shrink;
110	struct ttm_pool_opts	options;
111
112	union {
113		struct ttm_page_pool	pools[NUM_POOLS];
114		struct {
115			struct ttm_page_pool	wc_pool;
116			struct ttm_page_pool	uc_pool;
117			struct ttm_page_pool	wc_pool_dma32;
118			struct ttm_page_pool	uc_pool_dma32;
119		} ;
120	};
121};
122
123static struct attribute ttm_page_pool_max = {
124	.name = "pool_max_size",
125	.mode = S_IRUGO | S_IWUSR
126};
127static struct attribute ttm_page_pool_small = {
128	.name = "pool_small_allocation",
129	.mode = S_IRUGO | S_IWUSR
130};
131static struct attribute ttm_page_pool_alloc_size = {
132	.name = "pool_allocation_size",
133	.mode = S_IRUGO | S_IWUSR
134};
135
136static struct attribute *ttm_pool_attrs[] = {
137	&ttm_page_pool_max,
138	&ttm_page_pool_small,
139	&ttm_page_pool_alloc_size,
140	NULL
141};
142
143static void ttm_pool_kobj_release(struct kobject *kobj)
144{
145	struct ttm_pool_manager *m =
146		container_of(kobj, struct ttm_pool_manager, kobj);
147	kfree(m);
148}
149
150static ssize_t ttm_pool_store(struct kobject *kobj,
151		struct attribute *attr, const char *buffer, size_t size)
152{
153	struct ttm_pool_manager *m =
154		container_of(kobj, struct ttm_pool_manager, kobj);
155	int chars;
156	unsigned val;
157	chars = sscanf(buffer, "%u", &val);
158	if (chars == 0)
159		return size;
160
161	/* Convert kb to number of pages */
162	val = val / (PAGE_SIZE >> 10);
163
164	if (attr == &ttm_page_pool_max)
165		m->options.max_size = val;
166	else if (attr == &ttm_page_pool_small)
167		m->options.small = val;
168	else if (attr == &ttm_page_pool_alloc_size) {
169		if (val > NUM_PAGES_TO_ALLOC*8) {
170			printk(KERN_ERR TTM_PFX
171			       "Setting allocation size to %lu "
172			       "is not allowed. Recommended size is "
173			       "%lu\n",
174			       NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
175			       NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
176			return size;
177		} else if (val > NUM_PAGES_TO_ALLOC) {
178			printk(KERN_WARNING TTM_PFX
179			       "Setting allocation size to "
180			       "larger than %lu is not recommended.\n",
181			       NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
182		}
183		m->options.alloc_size = val;
184	}
185
186	return size;
187}
188
189static ssize_t ttm_pool_show(struct kobject *kobj,
190		struct attribute *attr, char *buffer)
191{
192	struct ttm_pool_manager *m =
193		container_of(kobj, struct ttm_pool_manager, kobj);
194	unsigned val = 0;
195
196	if (attr == &ttm_page_pool_max)
197		val = m->options.max_size;
198	else if (attr == &ttm_page_pool_small)
199		val = m->options.small;
200	else if (attr == &ttm_page_pool_alloc_size)
201		val = m->options.alloc_size;
202
203	val = val * (PAGE_SIZE >> 10);
204
205	return snprintf(buffer, PAGE_SIZE, "%u\n", val);
206}
207
208static const struct sysfs_ops ttm_pool_sysfs_ops = {
209	.show = &ttm_pool_show,
210	.store = &ttm_pool_store,
211};
212
213static struct kobj_type ttm_pool_kobj_type = {
214	.release = &ttm_pool_kobj_release,
215	.sysfs_ops = &ttm_pool_sysfs_ops,
216	.default_attrs = ttm_pool_attrs,
217};
218
219static struct ttm_pool_manager *_manager;
220
221#ifndef CONFIG_X86
222static int set_pages_array_wb(struct page **pages, int addrinarray)
223{
224#ifdef TTM_HAS_AGP
225	int i;
226
227	for (i = 0; i < addrinarray; i++)
228		unmap_page_from_agp(pages[i]);
229#endif
230	return 0;
231}
232
233static int set_pages_array_wc(struct page **pages, int addrinarray)
234{
235#ifdef TTM_HAS_AGP
236	int i;
237
238	for (i = 0; i < addrinarray; i++)
239		map_page_into_agp(pages[i]);
240#endif
241	return 0;
242}
243
244static int set_pages_array_uc(struct page **pages, int addrinarray)
245{
246#ifdef TTM_HAS_AGP
247	int i;
248
249	for (i = 0; i < addrinarray; i++)
250		map_page_into_agp(pages[i]);
251#endif
252	return 0;
253}
254#endif
255
256/**
257 * Select the right pool or requested caching state and ttm flags. */
258static struct ttm_page_pool *ttm_get_pool(int flags,
259		enum ttm_caching_state cstate)
260{
261	int pool_index;
262
263	if (cstate == tt_cached)
264		return NULL;
265
266	if (cstate == tt_wc)
267		pool_index = 0x0;
268	else
269		pool_index = 0x1;
270
271	if (flags & TTM_PAGE_FLAG_DMA32)
272		pool_index |= 0x2;
273
274	return &_manager->pools[pool_index];
275}
276
277/* set memory back to wb and free the pages. */
278static void ttm_pages_put(struct page *pages[], unsigned npages)
279{
280	unsigned i;
281	if (set_pages_array_wb(pages, npages))
282		printk(KERN_ERR TTM_PFX "Failed to set %d pages to wb!\n",
283				npages);
284	for (i = 0; i < npages; ++i)
285		__free_page(pages[i]);
286}
287
288static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
289		unsigned freed_pages)
290{
291	pool->npages -= freed_pages;
292	pool->nfrees += freed_pages;
293}
294
295/**
296 * Free pages from pool.
297 *
298 * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
299 * number of pages in one go.
300 *
301 * @pool: to free the pages from
302 * @free_all: If set to true will free all pages in pool
303 **/
304static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
305{
306	unsigned long irq_flags;
307	struct page *p;
308	struct page **pages_to_free;
309	unsigned freed_pages = 0,
310		 npages_to_free = nr_free;
311
312	if (NUM_PAGES_TO_ALLOC < nr_free)
313		npages_to_free = NUM_PAGES_TO_ALLOC;
314
315	pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
316			GFP_KERNEL);
317	if (!pages_to_free) {
318		printk(KERN_ERR TTM_PFX
319		       "Failed to allocate memory for pool free operation.\n");
320		return 0;
321	}
322
323restart:
324	spin_lock_irqsave(&pool->lock, irq_flags);
325
326	list_for_each_entry_reverse(p, &pool->list, lru) {
327		if (freed_pages >= npages_to_free)
328			break;
329
330		pages_to_free[freed_pages++] = p;
331		/* We can only remove NUM_PAGES_TO_ALLOC at a time. */
332		if (freed_pages >= NUM_PAGES_TO_ALLOC) {
333			/* remove range of pages from the pool */
334			__list_del(p->lru.prev, &pool->list);
335
336			ttm_pool_update_free_locked(pool, freed_pages);
337			/**
338			 * Because changing page caching is costly
339			 * we unlock the pool to prevent stalling.
340			 */
341			spin_unlock_irqrestore(&pool->lock, irq_flags);
342
343			ttm_pages_put(pages_to_free, freed_pages);
344			if (likely(nr_free != FREE_ALL_PAGES))
345				nr_free -= freed_pages;
346
347			if (NUM_PAGES_TO_ALLOC >= nr_free)
348				npages_to_free = nr_free;
349			else
350				npages_to_free = NUM_PAGES_TO_ALLOC;
351
352			freed_pages = 0;
353
354			/* free all so restart the processing */
355			if (nr_free)
356				goto restart;
357
358			/* Not allowed to fall through or break because
359			 * following context is inside spinlock while we are
360			 * outside here.
361			 */
362			goto out;
363
364		}
365	}
366
367	/* remove range of pages from the pool */
368	if (freed_pages) {
369		__list_del(&p->lru, &pool->list);
370
371		ttm_pool_update_free_locked(pool, freed_pages);
372		nr_free -= freed_pages;
373	}
374
375	spin_unlock_irqrestore(&pool->lock, irq_flags);
376
377	if (freed_pages)
378		ttm_pages_put(pages_to_free, freed_pages);
379out:
380	kfree(pages_to_free);
381	return nr_free;
382}
383
384/* Get good estimation how many pages are free in pools */
385static int ttm_pool_get_num_unused_pages(void)
386{
387	unsigned i;
388	int total = 0;
389	for (i = 0; i < NUM_POOLS; ++i)
390		total += _manager->pools[i].npages;
391
392	return total;
393}
394
395/**
396 * Callback for mm to request pool to reduce number of page held.
397 */
398static int ttm_pool_mm_shrink(struct shrinker *shrink,
399			      struct shrink_control *sc)
400{
401	static atomic_t start_pool = ATOMIC_INIT(0);
402	unsigned i;
403	unsigned pool_offset = atomic_add_return(1, &start_pool);
404	struct ttm_page_pool *pool;
405	int shrink_pages = sc->nr_to_scan;
406
407	pool_offset = pool_offset % NUM_POOLS;
408	/* select start pool in round robin fashion */
409	for (i = 0; i < NUM_POOLS; ++i) {
410		unsigned nr_free = shrink_pages;
411		if (shrink_pages == 0)
412			break;
413		pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
414		shrink_pages = ttm_page_pool_free(pool, nr_free);
415	}
416	/* return estimated number of unused pages in pool */
417	return ttm_pool_get_num_unused_pages();
418}
419
420static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
421{
422	manager->mm_shrink.shrink = &ttm_pool_mm_shrink;
423	manager->mm_shrink.seeks = 1;
424	register_shrinker(&manager->mm_shrink);
425}
426
427static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
428{
429	unregister_shrinker(&manager->mm_shrink);
430}
431
432static int ttm_set_pages_caching(struct page **pages,
433		enum ttm_caching_state cstate, unsigned cpages)
434{
435	int r = 0;
436	/* Set page caching */
437	switch (cstate) {
438	case tt_uncached:
439		r = set_pages_array_uc(pages, cpages);
440		if (r)
441			printk(KERN_ERR TTM_PFX
442			       "Failed to set %d pages to uc!\n",
443			       cpages);
444		break;
445	case tt_wc:
446		r = set_pages_array_wc(pages, cpages);
447		if (r)
448			printk(KERN_ERR TTM_PFX
449			       "Failed to set %d pages to wc!\n",
450			       cpages);
451		break;
452	default:
453		break;
454	}
455	return r;
456}
457
458/**
459 * Free pages the pages that failed to change the caching state. If there is
460 * any pages that have changed their caching state already put them to the
461 * pool.
462 */
463static void ttm_handle_caching_state_failure(struct list_head *pages,
464		int ttm_flags, enum ttm_caching_state cstate,
465		struct page **failed_pages, unsigned cpages)
466{
467	unsigned i;
468	/* Failed pages have to be freed */
469	for (i = 0; i < cpages; ++i) {
470		list_del(&failed_pages[i]->lru);
471		__free_page(failed_pages[i]);
472	}
473}
474
475/**
476 * Allocate new pages with correct caching.
477 *
478 * This function is reentrant if caller updates count depending on number of
479 * pages returned in pages array.
480 */
481static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
482		int ttm_flags, enum ttm_caching_state cstate, unsigned count)
483{
484	struct page **caching_array;
485	struct page *p;
486	int r = 0;
487	unsigned i, cpages;
488	unsigned max_cpages = min(count,
489			(unsigned)(PAGE_SIZE/sizeof(struct page *)));
490
491	/* allocate array for page caching change */
492	caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
493
494	if (!caching_array) {
495		printk(KERN_ERR TTM_PFX
496		       "Unable to allocate table for new pages.");
497		return -ENOMEM;
498	}
499
500	for (i = 0, cpages = 0; i < count; ++i) {
501		p = alloc_page(gfp_flags);
502
503		if (!p) {
504			printk(KERN_ERR TTM_PFX "Unable to get page %u.\n", i);
505
506			/* store already allocated pages in the pool after
507			 * setting the caching state */
508			if (cpages) {
509				r = ttm_set_pages_caching(caching_array,
510							  cstate, cpages);
511				if (r)
512					ttm_handle_caching_state_failure(pages,
513						ttm_flags, cstate,
514						caching_array, cpages);
515			}
516			r = -ENOMEM;
517			goto out;
518		}
519
520#ifdef CONFIG_HIGHMEM
521		/* gfp flags of highmem page should never be dma32 so we
522		 * we should be fine in such case
523		 */
524		if (!PageHighMem(p))
525#endif
526		{
527			caching_array[cpages++] = p;
528			if (cpages == max_cpages) {
529
530				r = ttm_set_pages_caching(caching_array,
531						cstate, cpages);
532				if (r) {
533					ttm_handle_caching_state_failure(pages,
534						ttm_flags, cstate,
535						caching_array, cpages);
536					goto out;
537				}
538				cpages = 0;
539			}
540		}
541
542		list_add(&p->lru, pages);
543	}
544
545	if (cpages) {
546		r = ttm_set_pages_caching(caching_array, cstate, cpages);
547		if (r)
548			ttm_handle_caching_state_failure(pages,
549					ttm_flags, cstate,
550					caching_array, cpages);
551	}
552out:
553	kfree(caching_array);
554
555	return r;
556}
557
558/**
559 * Fill the given pool if there aren't enough pages and the requested number of
560 * pages is small.
561 */
562static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
563		int ttm_flags, enum ttm_caching_state cstate, unsigned count,
564		unsigned long *irq_flags)
565{
566	struct page *p;
567	int r;
568	unsigned cpages = 0;
569	/**
570	 * Only allow one pool fill operation at a time.
571	 * If pool doesn't have enough pages for the allocation new pages are
572	 * allocated from outside of pool.
573	 */
574	if (pool->fill_lock)
575		return;
576
577	pool->fill_lock = true;
578
579	/* If allocation request is small and there are not enough
580	 * pages in a pool we fill the pool up first. */
581	if (count < _manager->options.small
582		&& count > pool->npages) {
583		struct list_head new_pages;
584		unsigned alloc_size = _manager->options.alloc_size;
585
586		/**
587		 * Can't change page caching if in irqsave context. We have to
588		 * drop the pool->lock.
589		 */
590		spin_unlock_irqrestore(&pool->lock, *irq_flags);
591
592		INIT_LIST_HEAD(&new_pages);
593		r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
594				cstate,	alloc_size);
595		spin_lock_irqsave(&pool->lock, *irq_flags);
596
597		if (!r) {
598			list_splice(&new_pages, &pool->list);
599			++pool->nrefills;
600			pool->npages += alloc_size;
601		} else {
602			printk(KERN_ERR TTM_PFX
603			       "Failed to fill pool (%p).", pool);
604			/* If we have any pages left put them to the pool. */
605			list_for_each_entry(p, &pool->list, lru) {
606				++cpages;
607			}
608			list_splice(&new_pages, &pool->list);
609			pool->npages += cpages;
610		}
611
612	}
613	pool->fill_lock = false;
614}
615
616/**
617 * Cut 'count' number of pages from the pool and put them on the return list.
618 *
619 * @return count of pages still required to fulfill the request.
620 */
621static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
622		struct list_head *pages, int ttm_flags,
623		enum ttm_caching_state cstate, unsigned count)
624{
625	unsigned long irq_flags;
626	struct list_head *p;
627	unsigned i;
628
629	spin_lock_irqsave(&pool->lock, irq_flags);
630	ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
631
632	if (count >= pool->npages) {
633		/* take all pages from the pool */
634		list_splice_init(&pool->list, pages);
635		count -= pool->npages;
636		pool->npages = 0;
637		goto out;
638	}
639	/* find the last pages to include for requested number of pages. Split
640	 * pool to begin and halve it to reduce search space. */
641	if (count <= pool->npages/2) {
642		i = 0;
643		list_for_each(p, &pool->list) {
644			if (++i == count)
645				break;
646		}
647	} else {
648		i = pool->npages + 1;
649		list_for_each_prev(p, &pool->list) {
650			if (--i == count)
651				break;
652		}
653	}
654	/* Cut 'count' number of pages from the pool */
655	list_cut_position(pages, &pool->list, p);
656	pool->npages -= count;
657	count = 0;
658out:
659	spin_unlock_irqrestore(&pool->lock, irq_flags);
660	return count;
661}
662
663/*
664 * On success pages list will hold count number of correctly
665 * cached pages.
666 */
667int ttm_get_pages(struct list_head *pages, int flags,
668		  enum ttm_caching_state cstate, unsigned count,
669		  dma_addr_t *dma_address)
670{
671	struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
672	struct page *p = NULL;
673	gfp_t gfp_flags = GFP_USER;
674	int r;
675
676	/* set zero flag for page allocation if required */
677	if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
678		gfp_flags |= __GFP_ZERO;
679
680	/* No pool for cached pages */
681	if (pool == NULL) {
682		if (flags & TTM_PAGE_FLAG_DMA32)
683			gfp_flags |= GFP_DMA32;
684		else
685			gfp_flags |= GFP_HIGHUSER;
686
687		for (r = 0; r < count; ++r) {
688			p = alloc_page(gfp_flags);
689			if (!p) {
690
691				printk(KERN_ERR TTM_PFX
692				       "Unable to allocate page.");
693				return -ENOMEM;
694			}
695
696			list_add(&p->lru, pages);
697		}
698		return 0;
699	}
700
701
702	/* combine zero flag to pool flags */
703	gfp_flags |= pool->gfp_flags;
704
705	/* First we take pages from the pool */
706	count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count);
707
708	/* clear the pages coming from the pool if requested */
709	if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
710		list_for_each_entry(p, pages, lru) {
711			clear_page(page_address(p));
712		}
713	}
714
715	/* If pool didn't have enough pages allocate new one. */
716	if (count > 0) {
717		/* ttm_alloc_new_pages doesn't reference pool so we can run
718		 * multiple requests in parallel.
719		 **/
720		r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count);
721		if (r) {
722			/* If there is any pages in the list put them back to
723			 * the pool. */
724			printk(KERN_ERR TTM_PFX
725			       "Failed to allocate extra pages "
726			       "for large request.");
727			ttm_put_pages(pages, 0, flags, cstate, NULL);
728			return r;
729		}
730	}
731
732
733	return 0;
734}
735
736/* Put all pages in pages list to correct pool to wait for reuse */
737void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
738		   enum ttm_caching_state cstate, dma_addr_t *dma_address)
739{
740	unsigned long irq_flags;
741	struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
742	struct page *p, *tmp;
743
744	if (pool == NULL) {
745		/* No pool for this memory type so free the pages */
746
747		list_for_each_entry_safe(p, tmp, pages, lru) {
748			__free_page(p);
749		}
750		/* Make the pages list empty */
751		INIT_LIST_HEAD(pages);
752		return;
753	}
754	if (page_count == 0) {
755		list_for_each_entry_safe(p, tmp, pages, lru) {
756			++page_count;
757		}
758	}
759
760	spin_lock_irqsave(&pool->lock, irq_flags);
761	list_splice_init(pages, &pool->list);
762	pool->npages += page_count;
763	/* Check that we don't go over the pool limit */
764	page_count = 0;
765	if (pool->npages > _manager->options.max_size) {
766		page_count = pool->npages - _manager->options.max_size;
767		/* free at least NUM_PAGES_TO_ALLOC number of pages
768		 * to reduce calls to set_memory_wb */
769		if (page_count < NUM_PAGES_TO_ALLOC)
770			page_count = NUM_PAGES_TO_ALLOC;
771	}
772	spin_unlock_irqrestore(&pool->lock, irq_flags);
773	if (page_count)
774		ttm_page_pool_free(pool, page_count);
775}
776
777static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
778		char *name)
779{
780	spin_lock_init(&pool->lock);
781	pool->fill_lock = false;
782	INIT_LIST_HEAD(&pool->list);
783	pool->npages = pool->nfrees = 0;
784	pool->gfp_flags = flags;
785	pool->name = name;
786}
787
788int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
789{
790	int ret;
791
792	WARN_ON(_manager);
793
794	printk(KERN_INFO TTM_PFX "Initializing pool allocator.\n");
795
796	_manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
797
798	ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc");
799
800	ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc");
801
802	ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
803				  GFP_USER | GFP_DMA32, "wc dma");
804
805	ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
806				  GFP_USER | GFP_DMA32, "uc dma");
807
808	_manager->options.max_size = max_pages;
809	_manager->options.small = SMALL_ALLOCATION;
810	_manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
811
812	ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
813				   &glob->kobj, "pool");
814	if (unlikely(ret != 0)) {
815		kobject_put(&_manager->kobj);
816		_manager = NULL;
817		return ret;
818	}
819
820	ttm_pool_mm_shrink_init(_manager);
821
822	return 0;
823}
824
825void ttm_page_alloc_fini(void)
826{
827	int i;
828
829	printk(KERN_INFO TTM_PFX "Finalizing pool allocator.\n");
830	ttm_pool_mm_shrink_fini(_manager);
831
832	for (i = 0; i < NUM_POOLS; ++i)
833		ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES);
834
835	kobject_put(&_manager->kobj);
836	_manager = NULL;
837}
838
839int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
840{
841	struct ttm_page_pool *p;
842	unsigned i;
843	char *h[] = {"pool", "refills", "pages freed", "size"};
844	if (!_manager) {
845		seq_printf(m, "No pool allocator running.\n");
846		return 0;
847	}
848	seq_printf(m, "%6s %12s %13s %8s\n",
849			h[0], h[1], h[2], h[3]);
850	for (i = 0; i < NUM_POOLS; ++i) {
851		p = &_manager->pools[i];
852
853		seq_printf(m, "%6s %12ld %13ld %8d\n",
854				p->name, p->nrefills,
855				p->nfrees, p->npages);
856	}
857	return 0;
858}
859EXPORT_SYMBOL(ttm_page_alloc_debugfs);