Loading...
1/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 */
14
15#include <linux/sched.h>
16#include <linux/kernel.h>
17#include <linux/errno.h>
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/highmem.h>
21#include <linux/slab.h>
22#include <linux/pagemap.h>
23#include <linux/spinlock.h>
24#include <linux/cpumask.h>
25#include <linux/module.h>
26#include <linux/io.h>
27#include <linux/vmalloc.h>
28#include <linux/smp.h>
29
30#include <asm/pgtable.h>
31#include <asm/pgalloc.h>
32#include <asm/fixmap.h>
33#include <asm/tlb.h>
34#include <asm/tlbflush.h>
35#include <asm/homecache.h>
36
37#define K(x) ((x) << (PAGE_SHIFT-10))
38
39/*
40 * The normal show_free_areas() is too verbose on Tile, with dozens
41 * of processors and often four NUMA zones each with high and lowmem.
42 */
43void show_mem(unsigned int filter)
44{
45 struct zone *zone;
46
47 pr_err("Active:%lu inactive:%lu dirty:%lu writeback:%lu unstable:%lu"
48 " free:%lu\n slab:%lu mapped:%lu pagetables:%lu bounce:%lu"
49 " pagecache:%lu swap:%lu\n",
50 (global_page_state(NR_ACTIVE_ANON) +
51 global_page_state(NR_ACTIVE_FILE)),
52 (global_page_state(NR_INACTIVE_ANON) +
53 global_page_state(NR_INACTIVE_FILE)),
54 global_page_state(NR_FILE_DIRTY),
55 global_page_state(NR_WRITEBACK),
56 global_page_state(NR_UNSTABLE_NFS),
57 global_page_state(NR_FREE_PAGES),
58 (global_page_state(NR_SLAB_RECLAIMABLE) +
59 global_page_state(NR_SLAB_UNRECLAIMABLE)),
60 global_page_state(NR_FILE_MAPPED),
61 global_page_state(NR_PAGETABLE),
62 global_page_state(NR_BOUNCE),
63 global_page_state(NR_FILE_PAGES),
64 nr_swap_pages);
65
66 for_each_zone(zone) {
67 unsigned long flags, order, total = 0, largest_order = -1;
68
69 if (!populated_zone(zone))
70 continue;
71
72 spin_lock_irqsave(&zone->lock, flags);
73 for (order = 0; order < MAX_ORDER; order++) {
74 int nr = zone->free_area[order].nr_free;
75 total += nr << order;
76 if (nr)
77 largest_order = order;
78 }
79 spin_unlock_irqrestore(&zone->lock, flags);
80 pr_err("Node %d %7s: %lukB (largest %luKb)\n",
81 zone_to_nid(zone), zone->name,
82 K(total), largest_order ? K(1UL) << largest_order : 0);
83 }
84}
85
86/*
87 * Associate a virtual page frame with a given physical page frame
88 * and protection flags for that frame.
89 */
90static void set_pte_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
91{
92 pgd_t *pgd;
93 pud_t *pud;
94 pmd_t *pmd;
95 pte_t *pte;
96
97 pgd = swapper_pg_dir + pgd_index(vaddr);
98 if (pgd_none(*pgd)) {
99 BUG();
100 return;
101 }
102 pud = pud_offset(pgd, vaddr);
103 if (pud_none(*pud)) {
104 BUG();
105 return;
106 }
107 pmd = pmd_offset(pud, vaddr);
108 if (pmd_none(*pmd)) {
109 BUG();
110 return;
111 }
112 pte = pte_offset_kernel(pmd, vaddr);
113 /* <pfn,flags> stored as-is, to permit clearing entries */
114 set_pte(pte, pfn_pte(pfn, flags));
115
116 /*
117 * It's enough to flush this one mapping.
118 * This appears conservative since it is only called
119 * from __set_fixmap.
120 */
121 local_flush_tlb_page(NULL, vaddr, PAGE_SIZE);
122}
123
124void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
125{
126 unsigned long address = __fix_to_virt(idx);
127
128 if (idx >= __end_of_fixed_addresses) {
129 BUG();
130 return;
131 }
132 set_pte_pfn(address, phys >> PAGE_SHIFT, flags);
133}
134
135/**
136 * shatter_huge_page() - ensure a given address is mapped by a small page.
137 *
138 * This function converts a huge PTE mapping kernel LOWMEM into a bunch
139 * of small PTEs with the same caching. No cache flush required, but we
140 * must do a global TLB flush.
141 *
142 * Any caller that wishes to modify a kernel mapping that might
143 * have been made with a huge page should call this function,
144 * since doing so properly avoids race conditions with installing the
145 * newly-shattered page and then flushing all the TLB entries.
146 *
147 * @addr: Address at which to shatter any existing huge page.
148 */
149void shatter_huge_page(unsigned long addr)
150{
151 pgd_t *pgd;
152 pud_t *pud;
153 pmd_t *pmd;
154 unsigned long flags = 0; /* happy compiler */
155#ifdef __PAGETABLE_PMD_FOLDED
156 struct list_head *pos;
157#endif
158
159 /* Get a pointer to the pmd entry that we need to change. */
160 addr &= HPAGE_MASK;
161 BUG_ON(pgd_addr_invalid(addr));
162 BUG_ON(addr < PAGE_OFFSET); /* only for kernel LOWMEM */
163 pgd = swapper_pg_dir + pgd_index(addr);
164 pud = pud_offset(pgd, addr);
165 BUG_ON(!pud_present(*pud));
166 pmd = pmd_offset(pud, addr);
167 BUG_ON(!pmd_present(*pmd));
168 if (!pmd_huge_page(*pmd))
169 return;
170
171 spin_lock_irqsave(&init_mm.page_table_lock, flags);
172 if (!pmd_huge_page(*pmd)) {
173 /* Lost the race to convert the huge page. */
174 spin_unlock_irqrestore(&init_mm.page_table_lock, flags);
175 return;
176 }
177
178 /* Shatter the huge page into the preallocated L2 page table. */
179 pmd_populate_kernel(&init_mm, pmd,
180 get_prealloc_pte(pte_pfn(*(pte_t *)pmd)));
181
182#ifdef __PAGETABLE_PMD_FOLDED
183 /* Walk every pgd on the system and update the pmd there. */
184 spin_lock(&pgd_lock);
185 list_for_each(pos, &pgd_list) {
186 pmd_t *copy_pmd;
187 pgd = list_to_pgd(pos) + pgd_index(addr);
188 pud = pud_offset(pgd, addr);
189 copy_pmd = pmd_offset(pud, addr);
190 __set_pmd(copy_pmd, *pmd);
191 }
192 spin_unlock(&pgd_lock);
193#endif
194
195 /* Tell every cpu to notice the change. */
196 flush_remote(0, 0, NULL, addr, HPAGE_SIZE, HPAGE_SIZE,
197 cpu_possible_mask, NULL, 0);
198
199 /* Hold the lock until the TLB flush is finished to avoid races. */
200 spin_unlock_irqrestore(&init_mm.page_table_lock, flags);
201}
202
203/*
204 * List of all pgd's needed so it can invalidate entries in both cached
205 * and uncached pgd's. This is essentially codepath-based locking
206 * against pageattr.c; it is the unique case in which a valid change
207 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
208 * vmalloc faults work because attached pagetables are never freed.
209 *
210 * The lock is always taken with interrupts disabled, unlike on x86
211 * and other platforms, because we need to take the lock in
212 * shatter_huge_page(), which may be called from an interrupt context.
213 * We are not at risk from the tlbflush IPI deadlock that was seen on
214 * x86, since we use the flush_remote() API to have the hypervisor do
215 * the TLB flushes regardless of irq disabling.
216 */
217DEFINE_SPINLOCK(pgd_lock);
218LIST_HEAD(pgd_list);
219
220static inline void pgd_list_add(pgd_t *pgd)
221{
222 list_add(pgd_to_list(pgd), &pgd_list);
223}
224
225static inline void pgd_list_del(pgd_t *pgd)
226{
227 list_del(pgd_to_list(pgd));
228}
229
230#define KERNEL_PGD_INDEX_START pgd_index(PAGE_OFFSET)
231#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_INDEX_START)
232
233static void pgd_ctor(pgd_t *pgd)
234{
235 unsigned long flags;
236
237 memset(pgd, 0, KERNEL_PGD_INDEX_START*sizeof(pgd_t));
238 spin_lock_irqsave(&pgd_lock, flags);
239
240#ifndef __tilegx__
241 /*
242 * Check that the user interrupt vector has no L2.
243 * It never should for the swapper, and new page tables
244 * should always start with an empty user interrupt vector.
245 */
246 BUG_ON(((u64 *)swapper_pg_dir)[pgd_index(MEM_USER_INTRPT)] != 0);
247#endif
248
249 memcpy(pgd + KERNEL_PGD_INDEX_START,
250 swapper_pg_dir + KERNEL_PGD_INDEX_START,
251 KERNEL_PGD_PTRS * sizeof(pgd_t));
252
253 pgd_list_add(pgd);
254 spin_unlock_irqrestore(&pgd_lock, flags);
255}
256
257static void pgd_dtor(pgd_t *pgd)
258{
259 unsigned long flags; /* can be called from interrupt context */
260
261 spin_lock_irqsave(&pgd_lock, flags);
262 pgd_list_del(pgd);
263 spin_unlock_irqrestore(&pgd_lock, flags);
264}
265
266pgd_t *pgd_alloc(struct mm_struct *mm)
267{
268 pgd_t *pgd = kmem_cache_alloc(pgd_cache, GFP_KERNEL);
269 if (pgd)
270 pgd_ctor(pgd);
271 return pgd;
272}
273
274void pgd_free(struct mm_struct *mm, pgd_t *pgd)
275{
276 pgd_dtor(pgd);
277 kmem_cache_free(pgd_cache, pgd);
278}
279
280
281#define L2_USER_PGTABLE_PAGES (1 << L2_USER_PGTABLE_ORDER)
282
283struct page *pgtable_alloc_one(struct mm_struct *mm, unsigned long address,
284 int order)
285{
286 gfp_t flags = GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO;
287 struct page *p;
288 int i;
289
290 p = alloc_pages(flags, L2_USER_PGTABLE_ORDER);
291 if (p == NULL)
292 return NULL;
293
294 /*
295 * Make every page have a page_count() of one, not just the first.
296 * We don't use __GFP_COMP since it doesn't look like it works
297 * correctly with tlb_remove_page().
298 */
299 for (i = 1; i < order; ++i) {
300 init_page_count(p+i);
301 inc_zone_page_state(p+i, NR_PAGETABLE);
302 }
303
304 pgtable_page_ctor(p);
305 return p;
306}
307
308/*
309 * Free page immediately (used in __pte_alloc if we raced with another
310 * process). We have to correct whatever pte_alloc_one() did before
311 * returning the pages to the allocator.
312 */
313void pgtable_free(struct mm_struct *mm, struct page *p, int order)
314{
315 int i;
316
317 pgtable_page_dtor(p);
318 __free_page(p);
319
320 for (i = 1; i < order; ++i) {
321 __free_page(p+i);
322 dec_zone_page_state(p+i, NR_PAGETABLE);
323 }
324}
325
326void __pgtable_free_tlb(struct mmu_gather *tlb, struct page *pte,
327 unsigned long address, int order)
328{
329 int i;
330
331 pgtable_page_dtor(pte);
332 tlb_remove_page(tlb, pte);
333
334 for (i = 1; i < order; ++i) {
335 tlb_remove_page(tlb, pte + i);
336 dec_zone_page_state(pte + i, NR_PAGETABLE);
337 }
338}
339
340#ifndef __tilegx__
341
342/*
343 * FIXME: needs to be atomic vs hypervisor writes. For now we make the
344 * window of vulnerability a bit smaller by doing an unlocked 8-bit update.
345 */
346int ptep_test_and_clear_young(struct vm_area_struct *vma,
347 unsigned long addr, pte_t *ptep)
348{
349#if HV_PTE_INDEX_ACCESSED < 8 || HV_PTE_INDEX_ACCESSED >= 16
350# error Code assumes HV_PTE "accessed" bit in second byte
351#endif
352 u8 *tmp = (u8 *)ptep;
353 u8 second_byte = tmp[1];
354 if (!(second_byte & (1 << (HV_PTE_INDEX_ACCESSED - 8))))
355 return 0;
356 tmp[1] = second_byte & ~(1 << (HV_PTE_INDEX_ACCESSED - 8));
357 return 1;
358}
359
360/*
361 * This implementation is atomic vs hypervisor writes, since the hypervisor
362 * always writes the low word (where "accessed" and "dirty" are) and this
363 * routine only writes the high word.
364 */
365void ptep_set_wrprotect(struct mm_struct *mm,
366 unsigned long addr, pte_t *ptep)
367{
368#if HV_PTE_INDEX_WRITABLE < 32
369# error Code assumes HV_PTE "writable" bit in high word
370#endif
371 u32 *tmp = (u32 *)ptep;
372 tmp[1] = tmp[1] & ~(1 << (HV_PTE_INDEX_WRITABLE - 32));
373}
374
375#endif
376
377pte_t *virt_to_pte(struct mm_struct* mm, unsigned long addr)
378{
379 pgd_t *pgd;
380 pud_t *pud;
381 pmd_t *pmd;
382
383 if (pgd_addr_invalid(addr))
384 return NULL;
385
386 pgd = mm ? pgd_offset(mm, addr) : swapper_pg_dir + pgd_index(addr);
387 pud = pud_offset(pgd, addr);
388 if (!pud_present(*pud))
389 return NULL;
390 pmd = pmd_offset(pud, addr);
391 if (pmd_huge_page(*pmd))
392 return (pte_t *)pmd;
393 if (!pmd_present(*pmd))
394 return NULL;
395 return pte_offset_kernel(pmd, addr);
396}
397
398pgprot_t set_remote_cache_cpu(pgprot_t prot, int cpu)
399{
400 unsigned int width = smp_width;
401 int x = cpu % width;
402 int y = cpu / width;
403 BUG_ON(y >= smp_height);
404 BUG_ON(hv_pte_get_mode(prot) != HV_PTE_MODE_CACHE_TILE_L3);
405 BUG_ON(cpu < 0 || cpu >= NR_CPUS);
406 BUG_ON(!cpu_is_valid_lotar(cpu));
407 return hv_pte_set_lotar(prot, HV_XY_TO_LOTAR(x, y));
408}
409
410int get_remote_cache_cpu(pgprot_t prot)
411{
412 HV_LOTAR lotar = hv_pte_get_lotar(prot);
413 int x = HV_LOTAR_X(lotar);
414 int y = HV_LOTAR_Y(lotar);
415 BUG_ON(hv_pte_get_mode(prot) != HV_PTE_MODE_CACHE_TILE_L3);
416 return x + y * smp_width;
417}
418
419/*
420 * Convert a kernel VA to a PA and homing information.
421 */
422int va_to_cpa_and_pte(void *va, unsigned long long *cpa, pte_t *pte)
423{
424 struct page *page = virt_to_page(va);
425 pte_t null_pte = { 0 };
426
427 *cpa = __pa(va);
428
429 /* Note that this is not writing a page table, just returning a pte. */
430 *pte = pte_set_home(null_pte, page_home(page));
431
432 return 0; /* return non-zero if not hfh? */
433}
434EXPORT_SYMBOL(va_to_cpa_and_pte);
435
436void __set_pte(pte_t *ptep, pte_t pte)
437{
438#ifdef __tilegx__
439 *ptep = pte;
440#else
441# if HV_PTE_INDEX_PRESENT >= 32 || HV_PTE_INDEX_MIGRATING >= 32
442# error Must write the present and migrating bits last
443# endif
444 if (pte_present(pte)) {
445 ((u32 *)ptep)[1] = (u32)(pte_val(pte) >> 32);
446 barrier();
447 ((u32 *)ptep)[0] = (u32)(pte_val(pte));
448 } else {
449 ((u32 *)ptep)[0] = (u32)(pte_val(pte));
450 barrier();
451 ((u32 *)ptep)[1] = (u32)(pte_val(pte) >> 32);
452 }
453#endif /* __tilegx__ */
454}
455
456void set_pte(pte_t *ptep, pte_t pte)
457{
458 if (pte_present(pte) &&
459 (!CHIP_HAS_MMIO() || hv_pte_get_mode(pte) != HV_PTE_MODE_MMIO)) {
460 /* The PTE actually references physical memory. */
461 unsigned long pfn = pte_pfn(pte);
462 if (pfn_valid(pfn)) {
463 /* Update the home of the PTE from the struct page. */
464 pte = pte_set_home(pte, page_home(pfn_to_page(pfn)));
465 } else if (hv_pte_get_mode(pte) == 0) {
466 /* remap_pfn_range(), etc, must supply PTE mode. */
467 panic("set_pte(): out-of-range PFN and mode 0\n");
468 }
469 }
470
471 __set_pte(ptep, pte);
472}
473
474/* Can this mm load a PTE with cached_priority set? */
475static inline int mm_is_priority_cached(struct mm_struct *mm)
476{
477 return mm->context.priority_cached != 0;
478}
479
480/*
481 * Add a priority mapping to an mm_context and
482 * notify the hypervisor if this is the first one.
483 */
484void start_mm_caching(struct mm_struct *mm)
485{
486 if (!mm_is_priority_cached(mm)) {
487 mm->context.priority_cached = -1UL;
488 hv_set_caching(-1UL);
489 }
490}
491
492/*
493 * Validate and return the priority_cached flag. We know if it's zero
494 * that we don't need to scan, since we immediately set it non-zero
495 * when we first consider a MAP_CACHE_PRIORITY mapping.
496 *
497 * We only _try_ to acquire the mmap_sem semaphore; if we can't acquire it,
498 * since we're in an interrupt context (servicing switch_mm) we don't
499 * worry about it and don't unset the "priority_cached" field.
500 * Presumably we'll come back later and have more luck and clear
501 * the value then; for now we'll just keep the cache marked for priority.
502 */
503static unsigned long update_priority_cached(struct mm_struct *mm)
504{
505 if (mm->context.priority_cached && down_write_trylock(&mm->mmap_sem)) {
506 struct vm_area_struct *vm;
507 for (vm = mm->mmap; vm; vm = vm->vm_next) {
508 if (hv_pte_get_cached_priority(vm->vm_page_prot))
509 break;
510 }
511 if (vm == NULL)
512 mm->context.priority_cached = 0;
513 up_write(&mm->mmap_sem);
514 }
515 return mm->context.priority_cached;
516}
517
518/* Set caching correctly for an mm that we are switching to. */
519void check_mm_caching(struct mm_struct *prev, struct mm_struct *next)
520{
521 if (!mm_is_priority_cached(next)) {
522 /*
523 * If the new mm doesn't use priority caching, just see if we
524 * need the hv_set_caching(), or can assume it's already zero.
525 */
526 if (mm_is_priority_cached(prev))
527 hv_set_caching(0);
528 } else {
529 hv_set_caching(update_priority_cached(next));
530 }
531}
532
533#if CHIP_HAS_MMIO()
534
535/* Map an arbitrary MMIO address, homed according to pgprot, into VA space. */
536void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
537 pgprot_t home)
538{
539 void *addr;
540 struct vm_struct *area;
541 unsigned long offset, last_addr;
542 pgprot_t pgprot;
543
544 /* Don't allow wraparound or zero size */
545 last_addr = phys_addr + size - 1;
546 if (!size || last_addr < phys_addr)
547 return NULL;
548
549 /* Create a read/write, MMIO VA mapping homed at the requested shim. */
550 pgprot = PAGE_KERNEL;
551 pgprot = hv_pte_set_mode(pgprot, HV_PTE_MODE_MMIO);
552 pgprot = hv_pte_set_lotar(pgprot, hv_pte_get_lotar(home));
553
554 /*
555 * Mappings have to be page-aligned
556 */
557 offset = phys_addr & ~PAGE_MASK;
558 phys_addr &= PAGE_MASK;
559 size = PAGE_ALIGN(last_addr+1) - phys_addr;
560
561 /*
562 * Ok, go for it..
563 */
564 area = get_vm_area(size, VM_IOREMAP /* | other flags? */);
565 if (!area)
566 return NULL;
567 area->phys_addr = phys_addr;
568 addr = area->addr;
569 if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
570 phys_addr, pgprot)) {
571 remove_vm_area((void *)(PAGE_MASK & (unsigned long) addr));
572 return NULL;
573 }
574 return (__force void __iomem *) (offset + (char *)addr);
575}
576EXPORT_SYMBOL(ioremap_prot);
577
578/* Map a PCI MMIO bus address into VA space. */
579void __iomem *ioremap(resource_size_t phys_addr, unsigned long size)
580{
581 panic("ioremap for PCI MMIO is not supported");
582}
583EXPORT_SYMBOL(ioremap);
584
585/* Unmap an MMIO VA mapping. */
586void iounmap(volatile void __iomem *addr_in)
587{
588 volatile void __iomem *addr = (volatile void __iomem *)
589 (PAGE_MASK & (unsigned long __force)addr_in);
590#if 1
591 vunmap((void * __force)addr);
592#else
593 /* x86 uses this complicated flow instead of vunmap(). Is
594 * there any particular reason we should do the same? */
595 struct vm_struct *p, *o;
596
597 /* Use the vm area unlocked, assuming the caller
598 ensures there isn't another iounmap for the same address
599 in parallel. Reuse of the virtual address is prevented by
600 leaving it in the global lists until we're done with it.
601 cpa takes care of the direct mappings. */
602 read_lock(&vmlist_lock);
603 for (p = vmlist; p; p = p->next) {
604 if (p->addr == addr)
605 break;
606 }
607 read_unlock(&vmlist_lock);
608
609 if (!p) {
610 pr_err("iounmap: bad address %p\n", addr);
611 dump_stack();
612 return;
613 }
614
615 /* Finally remove it */
616 o = remove_vm_area((void *)addr);
617 BUG_ON(p != o || o == NULL);
618 kfree(p);
619#endif
620}
621EXPORT_SYMBOL(iounmap);
622
623#endif /* CHIP_HAS_MMIO() */
1/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 */
14
15#include <linux/sched.h>
16#include <linux/kernel.h>
17#include <linux/errno.h>
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/highmem.h>
21#include <linux/slab.h>
22#include <linux/pagemap.h>
23#include <linux/spinlock.h>
24#include <linux/cpumask.h>
25#include <linux/module.h>
26#include <linux/io.h>
27#include <linux/vmalloc.h>
28#include <linux/smp.h>
29
30#include <asm/pgtable.h>
31#include <asm/pgalloc.h>
32#include <asm/fixmap.h>
33#include <asm/tlb.h>
34#include <asm/tlbflush.h>
35#include <asm/homecache.h>
36
37#define K(x) ((x) << (PAGE_SHIFT-10))
38
39/*
40 * The normal show_free_areas() is too verbose on Tile, with dozens
41 * of processors and often four NUMA zones each with high and lowmem.
42 */
43void show_mem(unsigned int filter)
44{
45 struct zone *zone;
46
47 pr_err("Active:%lu inactive:%lu dirty:%lu writeback:%lu unstable:%lu"
48 " free:%lu\n slab:%lu mapped:%lu pagetables:%lu bounce:%lu"
49 " pagecache:%lu swap:%lu\n",
50 (global_page_state(NR_ACTIVE_ANON) +
51 global_page_state(NR_ACTIVE_FILE)),
52 (global_page_state(NR_INACTIVE_ANON) +
53 global_page_state(NR_INACTIVE_FILE)),
54 global_page_state(NR_FILE_DIRTY),
55 global_page_state(NR_WRITEBACK),
56 global_page_state(NR_UNSTABLE_NFS),
57 global_page_state(NR_FREE_PAGES),
58 (global_page_state(NR_SLAB_RECLAIMABLE) +
59 global_page_state(NR_SLAB_UNRECLAIMABLE)),
60 global_page_state(NR_FILE_MAPPED),
61 global_page_state(NR_PAGETABLE),
62 global_page_state(NR_BOUNCE),
63 global_page_state(NR_FILE_PAGES),
64 get_nr_swap_pages());
65
66 for_each_zone(zone) {
67 unsigned long flags, order, total = 0, largest_order = -1;
68
69 if (!populated_zone(zone))
70 continue;
71
72 spin_lock_irqsave(&zone->lock, flags);
73 for (order = 0; order < MAX_ORDER; order++) {
74 int nr = zone->free_area[order].nr_free;
75 total += nr << order;
76 if (nr)
77 largest_order = order;
78 }
79 spin_unlock_irqrestore(&zone->lock, flags);
80 pr_err("Node %d %7s: %lukB (largest %luKb)\n",
81 zone_to_nid(zone), zone->name,
82 K(total), largest_order ? K(1UL) << largest_order : 0);
83 }
84}
85
86/**
87 * shatter_huge_page() - ensure a given address is mapped by a small page.
88 *
89 * This function converts a huge PTE mapping kernel LOWMEM into a bunch
90 * of small PTEs with the same caching. No cache flush required, but we
91 * must do a global TLB flush.
92 *
93 * Any caller that wishes to modify a kernel mapping that might
94 * have been made with a huge page should call this function,
95 * since doing so properly avoids race conditions with installing the
96 * newly-shattered page and then flushing all the TLB entries.
97 *
98 * @addr: Address at which to shatter any existing huge page.
99 */
100void shatter_huge_page(unsigned long addr)
101{
102 pgd_t *pgd;
103 pud_t *pud;
104 pmd_t *pmd;
105 unsigned long flags = 0; /* happy compiler */
106#ifdef __PAGETABLE_PMD_FOLDED
107 struct list_head *pos;
108#endif
109
110 /* Get a pointer to the pmd entry that we need to change. */
111 addr &= HPAGE_MASK;
112 BUG_ON(pgd_addr_invalid(addr));
113 BUG_ON(addr < PAGE_OFFSET); /* only for kernel LOWMEM */
114 pgd = swapper_pg_dir + pgd_index(addr);
115 pud = pud_offset(pgd, addr);
116 BUG_ON(!pud_present(*pud));
117 pmd = pmd_offset(pud, addr);
118 BUG_ON(!pmd_present(*pmd));
119 if (!pmd_huge_page(*pmd))
120 return;
121
122 spin_lock_irqsave(&init_mm.page_table_lock, flags);
123 if (!pmd_huge_page(*pmd)) {
124 /* Lost the race to convert the huge page. */
125 spin_unlock_irqrestore(&init_mm.page_table_lock, flags);
126 return;
127 }
128
129 /* Shatter the huge page into the preallocated L2 page table. */
130 pmd_populate_kernel(&init_mm, pmd, get_prealloc_pte(pmd_pfn(*pmd)));
131
132#ifdef __PAGETABLE_PMD_FOLDED
133 /* Walk every pgd on the system and update the pmd there. */
134 spin_lock(&pgd_lock);
135 list_for_each(pos, &pgd_list) {
136 pmd_t *copy_pmd;
137 pgd = list_to_pgd(pos) + pgd_index(addr);
138 pud = pud_offset(pgd, addr);
139 copy_pmd = pmd_offset(pud, addr);
140 __set_pmd(copy_pmd, *pmd);
141 }
142 spin_unlock(&pgd_lock);
143#endif
144
145 /* Tell every cpu to notice the change. */
146 flush_remote(0, 0, NULL, addr, HPAGE_SIZE, HPAGE_SIZE,
147 cpu_possible_mask, NULL, 0);
148
149 /* Hold the lock until the TLB flush is finished to avoid races. */
150 spin_unlock_irqrestore(&init_mm.page_table_lock, flags);
151}
152
153/*
154 * List of all pgd's needed so it can invalidate entries in both cached
155 * and uncached pgd's. This is essentially codepath-based locking
156 * against pageattr.c; it is the unique case in which a valid change
157 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
158 * vmalloc faults work because attached pagetables are never freed.
159 *
160 * The lock is always taken with interrupts disabled, unlike on x86
161 * and other platforms, because we need to take the lock in
162 * shatter_huge_page(), which may be called from an interrupt context.
163 * We are not at risk from the tlbflush IPI deadlock that was seen on
164 * x86, since we use the flush_remote() API to have the hypervisor do
165 * the TLB flushes regardless of irq disabling.
166 */
167DEFINE_SPINLOCK(pgd_lock);
168LIST_HEAD(pgd_list);
169
170static inline void pgd_list_add(pgd_t *pgd)
171{
172 list_add(pgd_to_list(pgd), &pgd_list);
173}
174
175static inline void pgd_list_del(pgd_t *pgd)
176{
177 list_del(pgd_to_list(pgd));
178}
179
180#define KERNEL_PGD_INDEX_START pgd_index(PAGE_OFFSET)
181#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_INDEX_START)
182
183static void pgd_ctor(pgd_t *pgd)
184{
185 unsigned long flags;
186
187 memset(pgd, 0, KERNEL_PGD_INDEX_START*sizeof(pgd_t));
188 spin_lock_irqsave(&pgd_lock, flags);
189
190#ifndef __tilegx__
191 /*
192 * Check that the user interrupt vector has no L2.
193 * It never should for the swapper, and new page tables
194 * should always start with an empty user interrupt vector.
195 */
196 BUG_ON(((u64 *)swapper_pg_dir)[pgd_index(MEM_USER_INTRPT)] != 0);
197#endif
198
199 memcpy(pgd + KERNEL_PGD_INDEX_START,
200 swapper_pg_dir + KERNEL_PGD_INDEX_START,
201 KERNEL_PGD_PTRS * sizeof(pgd_t));
202
203 pgd_list_add(pgd);
204 spin_unlock_irqrestore(&pgd_lock, flags);
205}
206
207static void pgd_dtor(pgd_t *pgd)
208{
209 unsigned long flags; /* can be called from interrupt context */
210
211 spin_lock_irqsave(&pgd_lock, flags);
212 pgd_list_del(pgd);
213 spin_unlock_irqrestore(&pgd_lock, flags);
214}
215
216pgd_t *pgd_alloc(struct mm_struct *mm)
217{
218 pgd_t *pgd = kmem_cache_alloc(pgd_cache, GFP_KERNEL);
219 if (pgd)
220 pgd_ctor(pgd);
221 return pgd;
222}
223
224void pgd_free(struct mm_struct *mm, pgd_t *pgd)
225{
226 pgd_dtor(pgd);
227 kmem_cache_free(pgd_cache, pgd);
228}
229
230
231#define L2_USER_PGTABLE_PAGES (1 << L2_USER_PGTABLE_ORDER)
232
233struct page *pgtable_alloc_one(struct mm_struct *mm, unsigned long address,
234 int order)
235{
236 gfp_t flags = GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO;
237 struct page *p;
238 int i;
239
240 p = alloc_pages(flags, L2_USER_PGTABLE_ORDER);
241 if (p == NULL)
242 return NULL;
243
244 if (!pgtable_page_ctor(p)) {
245 __free_pages(p, L2_USER_PGTABLE_ORDER);
246 return NULL;
247 }
248
249 /*
250 * Make every page have a page_count() of one, not just the first.
251 * We don't use __GFP_COMP since it doesn't look like it works
252 * correctly with tlb_remove_page().
253 */
254 for (i = 1; i < order; ++i) {
255 init_page_count(p+i);
256 inc_zone_page_state(p+i, NR_PAGETABLE);
257 }
258
259 return p;
260}
261
262/*
263 * Free page immediately (used in __pte_alloc if we raced with another
264 * process). We have to correct whatever pte_alloc_one() did before
265 * returning the pages to the allocator.
266 */
267void pgtable_free(struct mm_struct *mm, struct page *p, int order)
268{
269 int i;
270
271 pgtable_page_dtor(p);
272 __free_page(p);
273
274 for (i = 1; i < order; ++i) {
275 __free_page(p+i);
276 dec_zone_page_state(p+i, NR_PAGETABLE);
277 }
278}
279
280void __pgtable_free_tlb(struct mmu_gather *tlb, struct page *pte,
281 unsigned long address, int order)
282{
283 int i;
284
285 pgtable_page_dtor(pte);
286 tlb_remove_page(tlb, pte);
287
288 for (i = 1; i < order; ++i) {
289 tlb_remove_page(tlb, pte + i);
290 dec_zone_page_state(pte + i, NR_PAGETABLE);
291 }
292}
293
294#ifndef __tilegx__
295
296/*
297 * FIXME: needs to be atomic vs hypervisor writes. For now we make the
298 * window of vulnerability a bit smaller by doing an unlocked 8-bit update.
299 */
300int ptep_test_and_clear_young(struct vm_area_struct *vma,
301 unsigned long addr, pte_t *ptep)
302{
303#if HV_PTE_INDEX_ACCESSED < 8 || HV_PTE_INDEX_ACCESSED >= 16
304# error Code assumes HV_PTE "accessed" bit in second byte
305#endif
306 u8 *tmp = (u8 *)ptep;
307 u8 second_byte = tmp[1];
308 if (!(second_byte & (1 << (HV_PTE_INDEX_ACCESSED - 8))))
309 return 0;
310 tmp[1] = second_byte & ~(1 << (HV_PTE_INDEX_ACCESSED - 8));
311 return 1;
312}
313
314/*
315 * This implementation is atomic vs hypervisor writes, since the hypervisor
316 * always writes the low word (where "accessed" and "dirty" are) and this
317 * routine only writes the high word.
318 */
319void ptep_set_wrprotect(struct mm_struct *mm,
320 unsigned long addr, pte_t *ptep)
321{
322#if HV_PTE_INDEX_WRITABLE < 32
323# error Code assumes HV_PTE "writable" bit in high word
324#endif
325 u32 *tmp = (u32 *)ptep;
326 tmp[1] = tmp[1] & ~(1 << (HV_PTE_INDEX_WRITABLE - 32));
327}
328
329#endif
330
331/*
332 * Return a pointer to the PTE that corresponds to the given
333 * address in the given page table. A NULL page table just uses
334 * the standard kernel page table; the preferred API in this case
335 * is virt_to_kpte().
336 *
337 * The returned pointer can point to a huge page in other levels
338 * of the page table than the bottom, if the huge page is present
339 * in the page table. For bottom-level PTEs, the returned pointer
340 * can point to a PTE that is either present or not.
341 */
342pte_t *virt_to_pte(struct mm_struct* mm, unsigned long addr)
343{
344 pgd_t *pgd;
345 pud_t *pud;
346 pmd_t *pmd;
347
348 if (pgd_addr_invalid(addr))
349 return NULL;
350
351 pgd = mm ? pgd_offset(mm, addr) : swapper_pg_dir + pgd_index(addr);
352 pud = pud_offset(pgd, addr);
353 if (!pud_present(*pud))
354 return NULL;
355 if (pud_huge_page(*pud))
356 return (pte_t *)pud;
357 pmd = pmd_offset(pud, addr);
358 if (!pmd_present(*pmd))
359 return NULL;
360 if (pmd_huge_page(*pmd))
361 return (pte_t *)pmd;
362 return pte_offset_kernel(pmd, addr);
363}
364EXPORT_SYMBOL(virt_to_pte);
365
366pte_t *virt_to_kpte(unsigned long kaddr)
367{
368 BUG_ON(kaddr < PAGE_OFFSET);
369 return virt_to_pte(NULL, kaddr);
370}
371EXPORT_SYMBOL(virt_to_kpte);
372
373pgprot_t set_remote_cache_cpu(pgprot_t prot, int cpu)
374{
375 unsigned int width = smp_width;
376 int x = cpu % width;
377 int y = cpu / width;
378 BUG_ON(y >= smp_height);
379 BUG_ON(hv_pte_get_mode(prot) != HV_PTE_MODE_CACHE_TILE_L3);
380 BUG_ON(cpu < 0 || cpu >= NR_CPUS);
381 BUG_ON(!cpu_is_valid_lotar(cpu));
382 return hv_pte_set_lotar(prot, HV_XY_TO_LOTAR(x, y));
383}
384
385int get_remote_cache_cpu(pgprot_t prot)
386{
387 HV_LOTAR lotar = hv_pte_get_lotar(prot);
388 int x = HV_LOTAR_X(lotar);
389 int y = HV_LOTAR_Y(lotar);
390 BUG_ON(hv_pte_get_mode(prot) != HV_PTE_MODE_CACHE_TILE_L3);
391 return x + y * smp_width;
392}
393
394/*
395 * Convert a kernel VA to a PA and homing information.
396 */
397int va_to_cpa_and_pte(void *va, unsigned long long *cpa, pte_t *pte)
398{
399 struct page *page = virt_to_page(va);
400 pte_t null_pte = { 0 };
401
402 *cpa = __pa(va);
403
404 /* Note that this is not writing a page table, just returning a pte. */
405 *pte = pte_set_home(null_pte, page_home(page));
406
407 return 0; /* return non-zero if not hfh? */
408}
409EXPORT_SYMBOL(va_to_cpa_and_pte);
410
411void __set_pte(pte_t *ptep, pte_t pte)
412{
413#ifdef __tilegx__
414 *ptep = pte;
415#else
416# if HV_PTE_INDEX_PRESENT >= 32 || HV_PTE_INDEX_MIGRATING >= 32
417# error Must write the present and migrating bits last
418# endif
419 if (pte_present(pte)) {
420 ((u32 *)ptep)[1] = (u32)(pte_val(pte) >> 32);
421 barrier();
422 ((u32 *)ptep)[0] = (u32)(pte_val(pte));
423 } else {
424 ((u32 *)ptep)[0] = (u32)(pte_val(pte));
425 barrier();
426 ((u32 *)ptep)[1] = (u32)(pte_val(pte) >> 32);
427 }
428#endif /* __tilegx__ */
429}
430
431void set_pte(pte_t *ptep, pte_t pte)
432{
433 if (pte_present(pte) &&
434 (!CHIP_HAS_MMIO() || hv_pte_get_mode(pte) != HV_PTE_MODE_MMIO)) {
435 /* The PTE actually references physical memory. */
436 unsigned long pfn = pte_pfn(pte);
437 if (pfn_valid(pfn)) {
438 /* Update the home of the PTE from the struct page. */
439 pte = pte_set_home(pte, page_home(pfn_to_page(pfn)));
440 } else if (hv_pte_get_mode(pte) == 0) {
441 /* remap_pfn_range(), etc, must supply PTE mode. */
442 panic("set_pte(): out-of-range PFN and mode 0\n");
443 }
444 }
445
446 __set_pte(ptep, pte);
447}
448
449/* Can this mm load a PTE with cached_priority set? */
450static inline int mm_is_priority_cached(struct mm_struct *mm)
451{
452 return mm->context.priority_cached != 0;
453}
454
455/*
456 * Add a priority mapping to an mm_context and
457 * notify the hypervisor if this is the first one.
458 */
459void start_mm_caching(struct mm_struct *mm)
460{
461 if (!mm_is_priority_cached(mm)) {
462 mm->context.priority_cached = -1UL;
463 hv_set_caching(-1UL);
464 }
465}
466
467/*
468 * Validate and return the priority_cached flag. We know if it's zero
469 * that we don't need to scan, since we immediately set it non-zero
470 * when we first consider a MAP_CACHE_PRIORITY mapping.
471 *
472 * We only _try_ to acquire the mmap_sem semaphore; if we can't acquire it,
473 * since we're in an interrupt context (servicing switch_mm) we don't
474 * worry about it and don't unset the "priority_cached" field.
475 * Presumably we'll come back later and have more luck and clear
476 * the value then; for now we'll just keep the cache marked for priority.
477 */
478static unsigned long update_priority_cached(struct mm_struct *mm)
479{
480 if (mm->context.priority_cached && down_write_trylock(&mm->mmap_sem)) {
481 struct vm_area_struct *vm;
482 for (vm = mm->mmap; vm; vm = vm->vm_next) {
483 if (hv_pte_get_cached_priority(vm->vm_page_prot))
484 break;
485 }
486 if (vm == NULL)
487 mm->context.priority_cached = 0;
488 up_write(&mm->mmap_sem);
489 }
490 return mm->context.priority_cached;
491}
492
493/* Set caching correctly for an mm that we are switching to. */
494void check_mm_caching(struct mm_struct *prev, struct mm_struct *next)
495{
496 if (!mm_is_priority_cached(next)) {
497 /*
498 * If the new mm doesn't use priority caching, just see if we
499 * need the hv_set_caching(), or can assume it's already zero.
500 */
501 if (mm_is_priority_cached(prev))
502 hv_set_caching(0);
503 } else {
504 hv_set_caching(update_priority_cached(next));
505 }
506}
507
508#if CHIP_HAS_MMIO()
509
510/* Map an arbitrary MMIO address, homed according to pgprot, into VA space. */
511void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
512 pgprot_t home)
513{
514 void *addr;
515 struct vm_struct *area;
516 unsigned long offset, last_addr;
517 pgprot_t pgprot;
518
519 /* Don't allow wraparound or zero size */
520 last_addr = phys_addr + size - 1;
521 if (!size || last_addr < phys_addr)
522 return NULL;
523
524 /* Create a read/write, MMIO VA mapping homed at the requested shim. */
525 pgprot = PAGE_KERNEL;
526 pgprot = hv_pte_set_mode(pgprot, HV_PTE_MODE_MMIO);
527 pgprot = hv_pte_set_lotar(pgprot, hv_pte_get_lotar(home));
528
529 /*
530 * Mappings have to be page-aligned
531 */
532 offset = phys_addr & ~PAGE_MASK;
533 phys_addr &= PAGE_MASK;
534 size = PAGE_ALIGN(last_addr+1) - phys_addr;
535
536 /*
537 * Ok, go for it..
538 */
539 area = get_vm_area(size, VM_IOREMAP /* | other flags? */);
540 if (!area)
541 return NULL;
542 area->phys_addr = phys_addr;
543 addr = area->addr;
544 if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
545 phys_addr, pgprot)) {
546 free_vm_area(area);
547 return NULL;
548 }
549 return (__force void __iomem *) (offset + (char *)addr);
550}
551EXPORT_SYMBOL(ioremap_prot);
552
553/* Unmap an MMIO VA mapping. */
554void iounmap(volatile void __iomem *addr_in)
555{
556 volatile void __iomem *addr = (volatile void __iomem *)
557 (PAGE_MASK & (unsigned long __force)addr_in);
558#if 1
559 vunmap((void * __force)addr);
560#else
561 /* x86 uses this complicated flow instead of vunmap(). Is
562 * there any particular reason we should do the same? */
563 struct vm_struct *p, *o;
564
565 /* Use the vm area unlocked, assuming the caller
566 ensures there isn't another iounmap for the same address
567 in parallel. Reuse of the virtual address is prevented by
568 leaving it in the global lists until we're done with it.
569 cpa takes care of the direct mappings. */
570 p = find_vm_area((void *)addr);
571
572 if (!p) {
573 pr_err("iounmap: bad address %p\n", addr);
574 dump_stack();
575 return;
576 }
577
578 /* Finally remove it */
579 o = remove_vm_area((void *)addr);
580 BUG_ON(p != o || o == NULL);
581 kfree(p);
582#endif
583}
584EXPORT_SYMBOL(iounmap);
585
586#endif /* CHIP_HAS_MMIO() */