1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef _ASM_X86_PGTABLE_H
   3#define _ASM_X86_PGTABLE_H
   4
   5#include <linux/mem_encrypt.h>
   6#include <asm/page.h>
 
 
   7#include <asm/pgtable_types.h>
   8
   9/*
  10 * Macro to mark a page protection value as UC-
  11 */
  12#define pgprot_noncached(prot)						\
  13	((boot_cpu_data.x86 > 3)					\
  14	 ? (__pgprot(pgprot_val(prot) |					\
  15		     cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))	\
  16	 : (prot))
  17
  18/*
  19 * Macros to add or remove encryption attribute
  20 */
  21#define pgprot_encrypted(prot)	__pgprot(__sme_set(pgprot_val(prot)))
  22#define pgprot_decrypted(prot)	__pgprot(__sme_clr(pgprot_val(prot)))
  23
  24#ifndef __ASSEMBLY__
  25#include <asm/x86_init.h>
  26
  27extern pgd_t early_top_pgt[PTRS_PER_PGD];
  28int __init __early_make_pgtable(unsigned long address, pmdval_t pmd);
  29
  30void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd);
  31void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user);
  32void ptdump_walk_pgd_level_checkwx(void);
  33
  34#ifdef CONFIG_DEBUG_WX
  35#define debug_checkwx() ptdump_walk_pgd_level_checkwx()
  36#else
  37#define debug_checkwx() do { } while (0)
  38#endif
  39
  40/*
  41 * ZERO_PAGE is a global shared page that is always zero: used
  42 * for zero-mapped memory areas etc..
  43 */
  44extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
  45	__visible;
  46#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
  47
  48extern spinlock_t pgd_lock;
  49extern struct list_head pgd_list;
  50
  51extern struct mm_struct *pgd_page_get_mm(struct page *page);
  52
  53extern pmdval_t early_pmd_flags;
  54
  55#ifdef CONFIG_PARAVIRT
  56#include <asm/paravirt.h>
  57#else  /* !CONFIG_PARAVIRT */
  58#define set_pte(ptep, pte)		native_set_pte(ptep, pte)
  59#define set_pte_at(mm, addr, ptep, pte)	native_set_pte_at(mm, addr, ptep, pte)
 
  60
  61#define set_pte_atomic(ptep, pte)					\
  62	native_set_pte_atomic(ptep, pte)
  63
  64#define set_pmd(pmdp, pmd)		native_set_pmd(pmdp, pmd)
  65
  66#ifndef __PAGETABLE_P4D_FOLDED
  67#define set_pgd(pgdp, pgd)		native_set_pgd(pgdp, pgd)
  68#define pgd_clear(pgd)			(pgtable_l5_enabled ? native_pgd_clear(pgd) : 0)
  69#endif
  70
  71#ifndef set_p4d
  72# define set_p4d(p4dp, p4d)		native_set_p4d(p4dp, p4d)
  73#endif
  74
  75#ifndef __PAGETABLE_PUD_FOLDED
  76#define p4d_clear(p4d)			native_p4d_clear(p4d)
 
  77#endif
  78
  79#ifndef set_pud
  80# define set_pud(pudp, pud)		native_set_pud(pudp, pud)
  81#endif
  82
  83#ifndef __PAGETABLE_PUD_FOLDED
  84#define pud_clear(pud)			native_pud_clear(pud)
  85#endif
  86
  87#define pte_clear(mm, addr, ptep)	native_pte_clear(mm, addr, ptep)
  88#define pmd_clear(pmd)			native_pmd_clear(pmd)
  89
 
 
 
 
 
  90#define pgd_val(x)	native_pgd_val(x)
  91#define __pgd(x)	native_make_pgd(x)
  92
  93#ifndef __PAGETABLE_P4D_FOLDED
  94#define p4d_val(x)	native_p4d_val(x)
  95#define __p4d(x)	native_make_p4d(x)
  96#endif
  97
  98#ifndef __PAGETABLE_PUD_FOLDED
  99#define pud_val(x)	native_pud_val(x)
 100#define __pud(x)	native_make_pud(x)
 101#endif
 102
 103#ifndef __PAGETABLE_PMD_FOLDED
 104#define pmd_val(x)	native_pmd_val(x)
 105#define __pmd(x)	native_make_pmd(x)
 106#endif
 107
 108#define pte_val(x)	native_pte_val(x)
 109#define __pte(x)	native_make_pte(x)
 110
 111#define arch_end_context_switch(prev)	do {} while(0)
 112
 113#endif	/* CONFIG_PARAVIRT */
 114
 115/*
 116 * The following only work if pte_present() is true.
 117 * Undefined behaviour if not..
 118 */
 119static inline int pte_dirty(pte_t pte)
 120{
 121	return pte_flags(pte) & _PAGE_DIRTY;
 122}
 123
 124
 125static inline u32 read_pkru(void)
 126{
 127	if (boot_cpu_has(X86_FEATURE_OSPKE))
 128		return __read_pkru();
 129	return 0;
 130}
 131
 132static inline void write_pkru(u32 pkru)
 133{
 134	if (boot_cpu_has(X86_FEATURE_OSPKE))
 135		__write_pkru(pkru);
 136}
 137
 138static inline int pte_young(pte_t pte)
 139{
 140	return pte_flags(pte) & _PAGE_ACCESSED;
 141}
 142
 143static inline int pmd_dirty(pmd_t pmd)
 144{
 145	return pmd_flags(pmd) & _PAGE_DIRTY;
 146}
 147
 148static inline int pmd_young(pmd_t pmd)
 149{
 150	return pmd_flags(pmd) & _PAGE_ACCESSED;
 151}
 152
 153static inline int pud_dirty(pud_t pud)
 154{
 155	return pud_flags(pud) & _PAGE_DIRTY;
 156}
 157
 158static inline int pud_young(pud_t pud)
 159{
 160	return pud_flags(pud) & _PAGE_ACCESSED;
 161}
 162
 163static inline int pte_write(pte_t pte)
 164{
 165	return pte_flags(pte) & _PAGE_RW;
 166}
 167
 168static inline int pte_huge(pte_t pte)
 169{
 170	return pte_flags(pte) & _PAGE_PSE;
 171}
 172
 173static inline int pte_global(pte_t pte)
 174{
 175	return pte_flags(pte) & _PAGE_GLOBAL;
 176}
 177
 178static inline int pte_exec(pte_t pte)
 179{
 180	return !(pte_flags(pte) & _PAGE_NX);
 181}
 182
 183static inline int pte_special(pte_t pte)
 184{
 185	return pte_flags(pte) & _PAGE_SPECIAL;
 186}
 187
 188static inline unsigned long pte_pfn(pte_t pte)
 189{
 190	return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT;
 191}
 192
 193static inline unsigned long pmd_pfn(pmd_t pmd)
 194{
 195	return (pmd_val(pmd) & pmd_pfn_mask(pmd)) >> PAGE_SHIFT;
 196}
 197
 198static inline unsigned long pud_pfn(pud_t pud)
 199{
 200	return (pud_val(pud) & pud_pfn_mask(pud)) >> PAGE_SHIFT;
 201}
 202
 203static inline unsigned long p4d_pfn(p4d_t p4d)
 204{
 205	return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT;
 206}
 207
 208static inline unsigned long pgd_pfn(pgd_t pgd)
 209{
 210	return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
 211}
 212
 213static inline int p4d_large(p4d_t p4d)
 214{
 215	/* No 512 GiB pages yet */
 216	return 0;
 217}
 218
 219#define pte_page(pte)	pfn_to_page(pte_pfn(pte))
 220
 221static inline int pmd_large(pmd_t pte)
 222{
 223	return pmd_flags(pte) & _PAGE_PSE;
 
 224}
 225
 226#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 227static inline int pmd_trans_huge(pmd_t pmd)
 228{
 229	return (pmd_val(pmd) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
 230}
 231
 232#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
 233static inline int pud_trans_huge(pud_t pud)
 234{
 235	return (pud_val(pud) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
 236}
 237#endif
 238
 239#define has_transparent_hugepage has_transparent_hugepage
 240static inline int has_transparent_hugepage(void)
 241{
 242	return boot_cpu_has(X86_FEATURE_PSE);
 243}
 244
 245#ifdef __HAVE_ARCH_PTE_DEVMAP
 246static inline int pmd_devmap(pmd_t pmd)
 247{
 248	return !!(pmd_val(pmd) & _PAGE_DEVMAP);
 249}
 250
 251#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
 252static inline int pud_devmap(pud_t pud)
 253{
 254	return !!(pud_val(pud) & _PAGE_DEVMAP);
 255}
 256#else
 257static inline int pud_devmap(pud_t pud)
 258{
 259	return 0;
 260}
 261#endif
 262
 263static inline int pgd_devmap(pgd_t pgd)
 264{
 265	return 0;
 266}
 267#endif
 268#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 269
 270static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
 271{
 272	pteval_t v = native_pte_val(pte);
 273
 274	return native_make_pte(v | set);
 275}
 276
 277static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
 278{
 279	pteval_t v = native_pte_val(pte);
 280
 281	return native_make_pte(v & ~clear);
 282}
 283
 284static inline pte_t pte_mkclean(pte_t pte)
 285{
 286	return pte_clear_flags(pte, _PAGE_DIRTY);
 287}
 288
 289static inline pte_t pte_mkold(pte_t pte)
 290{
 291	return pte_clear_flags(pte, _PAGE_ACCESSED);
 292}
 293
 294static inline pte_t pte_wrprotect(pte_t pte)
 295{
 296	return pte_clear_flags(pte, _PAGE_RW);
 297}
 298
 299static inline pte_t pte_mkexec(pte_t pte)
 300{
 301	return pte_clear_flags(pte, _PAGE_NX);
 302}
 303
 304static inline pte_t pte_mkdirty(pte_t pte)
 305{
 306	return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
 307}
 308
 309static inline pte_t pte_mkyoung(pte_t pte)
 310{
 311	return pte_set_flags(pte, _PAGE_ACCESSED);
 312}
 313
 314static inline pte_t pte_mkwrite(pte_t pte)
 315{
 316	return pte_set_flags(pte, _PAGE_RW);
 317}
 318
 319static inline pte_t pte_mkhuge(pte_t pte)
 320{
 321	return pte_set_flags(pte, _PAGE_PSE);
 322}
 323
 324static inline pte_t pte_clrhuge(pte_t pte)
 325{
 326	return pte_clear_flags(pte, _PAGE_PSE);
 327}
 328
 329static inline pte_t pte_mkglobal(pte_t pte)
 330{
 331	return pte_set_flags(pte, _PAGE_GLOBAL);
 332}
 333
 334static inline pte_t pte_clrglobal(pte_t pte)
 335{
 336	return pte_clear_flags(pte, _PAGE_GLOBAL);
 337}
 338
 339static inline pte_t pte_mkspecial(pte_t pte)
 340{
 341	return pte_set_flags(pte, _PAGE_SPECIAL);
 342}
 343
 344static inline pte_t pte_mkdevmap(pte_t pte)
 345{
 346	return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
 347}
 348
 349static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
 350{
 351	pmdval_t v = native_pmd_val(pmd);
 352
 353	return native_make_pmd(v | set);
 354}
 355
 356static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
 357{
 358	pmdval_t v = native_pmd_val(pmd);
 359
 360	return native_make_pmd(v & ~clear);
 361}
 362
 363static inline pmd_t pmd_mkold(pmd_t pmd)
 364{
 365	return pmd_clear_flags(pmd, _PAGE_ACCESSED);
 366}
 367
 368static inline pmd_t pmd_mkclean(pmd_t pmd)
 369{
 370	return pmd_clear_flags(pmd, _PAGE_DIRTY);
 371}
 372
 373static inline pmd_t pmd_wrprotect(pmd_t pmd)
 374{
 375	return pmd_clear_flags(pmd, _PAGE_RW);
 376}
 377
 378static inline pmd_t pmd_mkdirty(pmd_t pmd)
 379{
 380	return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
 381}
 382
 383static inline pmd_t pmd_mkdevmap(pmd_t pmd)
 384{
 385	return pmd_set_flags(pmd, _PAGE_DEVMAP);
 386}
 387
 388static inline pmd_t pmd_mkhuge(pmd_t pmd)
 389{
 390	return pmd_set_flags(pmd, _PAGE_PSE);
 391}
 392
 393static inline pmd_t pmd_mkyoung(pmd_t pmd)
 394{
 395	return pmd_set_flags(pmd, _PAGE_ACCESSED);
 396}
 397
 398static inline pmd_t pmd_mkwrite(pmd_t pmd)
 399{
 400	return pmd_set_flags(pmd, _PAGE_RW);
 401}
 402
 403static inline pmd_t pmd_mknotpresent(pmd_t pmd)
 404{
 405	return pmd_clear_flags(pmd, _PAGE_PRESENT | _PAGE_PROTNONE);
 406}
 407
 408static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
 409{
 410	pudval_t v = native_pud_val(pud);
 411
 412	return native_make_pud(v | set);
 413}
 414
 415static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
 416{
 417	pudval_t v = native_pud_val(pud);
 418
 419	return native_make_pud(v & ~clear);
 420}
 421
 422static inline pud_t pud_mkold(pud_t pud)
 423{
 424	return pud_clear_flags(pud, _PAGE_ACCESSED);
 425}
 426
 427static inline pud_t pud_mkclean(pud_t pud)
 428{
 429	return pud_clear_flags(pud, _PAGE_DIRTY);
 430}
 431
 432static inline pud_t pud_wrprotect(pud_t pud)
 433{
 434	return pud_clear_flags(pud, _PAGE_RW);
 435}
 436
 437static inline pud_t pud_mkdirty(pud_t pud)
 438{
 439	return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
 440}
 441
 442static inline pud_t pud_mkdevmap(pud_t pud)
 443{
 444	return pud_set_flags(pud, _PAGE_DEVMAP);
 445}
 446
 447static inline pud_t pud_mkhuge(pud_t pud)
 448{
 449	return pud_set_flags(pud, _PAGE_PSE);
 450}
 451
 452static inline pud_t pud_mkyoung(pud_t pud)
 453{
 454	return pud_set_flags(pud, _PAGE_ACCESSED);
 455}
 456
 457static inline pud_t pud_mkwrite(pud_t pud)
 458{
 459	return pud_set_flags(pud, _PAGE_RW);
 460}
 461
 462static inline pud_t pud_mknotpresent(pud_t pud)
 463{
 464	return pud_clear_flags(pud, _PAGE_PRESENT | _PAGE_PROTNONE);
 465}
 466
 467#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
 468static inline int pte_soft_dirty(pte_t pte)
 469{
 470	return pte_flags(pte) & _PAGE_SOFT_DIRTY;
 471}
 472
 473static inline int pmd_soft_dirty(pmd_t pmd)
 474{
 475	return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
 476}
 477
 478static inline int pud_soft_dirty(pud_t pud)
 479{
 480	return pud_flags(pud) & _PAGE_SOFT_DIRTY;
 481}
 482
 483static inline pte_t pte_mksoft_dirty(pte_t pte)
 484{
 485	return pte_set_flags(pte, _PAGE_SOFT_DIRTY);
 486}
 487
 488static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
 489{
 490	return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
 491}
 492
 493static inline pud_t pud_mksoft_dirty(pud_t pud)
 494{
 495	return pud_set_flags(pud, _PAGE_SOFT_DIRTY);
 496}
 497
 498static inline pte_t pte_clear_soft_dirty(pte_t pte)
 499{
 500	return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
 501}
 502
 503static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
 504{
 505	return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
 506}
 507
 508static inline pud_t pud_clear_soft_dirty(pud_t pud)
 509{
 510	return pud_clear_flags(pud, _PAGE_SOFT_DIRTY);
 511}
 512
 513#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
 514
 515/*
 516 * Mask out unsupported bits in a present pgprot.  Non-present pgprots
 517 * can use those bits for other purposes, so leave them be.
 518 */
 519static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
 520{
 521	pgprotval_t protval = pgprot_val(pgprot);
 522
 523	if (protval & _PAGE_PRESENT)
 524		protval &= __supported_pte_mask;
 525
 526	return protval;
 527}
 528
 529static inline pgprotval_t check_pgprot(pgprot_t pgprot)
 530{
 531	pgprotval_t massaged_val = massage_pgprot(pgprot);
 532
 533	/* mmdebug.h can not be included here because of dependencies */
 534#ifdef CONFIG_DEBUG_VM
 535	WARN_ONCE(pgprot_val(pgprot) != massaged_val,
 536		  "attempted to set unsupported pgprot: %016llx "
 537		  "bits: %016llx supported: %016llx\n",
 538		  (u64)pgprot_val(pgprot),
 539		  (u64)pgprot_val(pgprot) ^ massaged_val,
 540		  (u64)__supported_pte_mask);
 541#endif
 542
 543	return massaged_val;
 544}
 545
 546static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
 547{
 548	return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) |
 549		     check_pgprot(pgprot));
 550}
 551
 552static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
 553{
 554	return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) |
 555		     check_pgprot(pgprot));
 556}
 557
 558static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot)
 559{
 560	return __pud(((phys_addr_t)page_nr << PAGE_SHIFT) |
 561		     check_pgprot(pgprot));
 562}
 563
 564static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 565{
 566	pteval_t val = pte_val(pte);
 567
 568	/*
 569	 * Chop off the NX bit (if present), and add the NX portion of
 570	 * the newprot (if present):
 571	 */
 572	val &= _PAGE_CHG_MASK;
 573	val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
 574
 575	return __pte(val);
 576}
 577
 578static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
 579{
 580	pmdval_t val = pmd_val(pmd);
 581
 582	val &= _HPAGE_CHG_MASK;
 583	val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
 584
 585	return __pmd(val);
 586}
 587
 588/* mprotect needs to preserve PAT bits when updating vm_page_prot */
 589#define pgprot_modify pgprot_modify
 590static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
 591{
 592	pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
 593	pgprotval_t addbits = pgprot_val(newprot);
 594	return __pgprot(preservebits | addbits);
 595}
 596
 597#define pte_pgprot(x) __pgprot(pte_flags(x))
 598#define pmd_pgprot(x) __pgprot(pmd_flags(x))
 599#define pud_pgprot(x) __pgprot(pud_flags(x))
 600#define p4d_pgprot(x) __pgprot(p4d_flags(x))
 601
 602#define canon_pgprot(p) __pgprot(massage_pgprot(p))
 603
 604static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
 605{
 606	return canon_pgprot(prot);
 607}
 608
 609static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
 610					 enum page_cache_mode pcm,
 611					 enum page_cache_mode new_pcm)
 612{
 613	/*
 614	 * PAT type is always WB for untracked ranges, so no need to check.
 615	 */
 616	if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
 617		return 1;
 618
 619	/*
 620	 * Certain new memtypes are not allowed with certain
 621	 * requested memtype:
 622	 * - request is uncached, return cannot be write-back
 623	 * - request is write-combine, return cannot be write-back
 624	 * - request is write-through, return cannot be write-back
 625	 * - request is write-through, return cannot be write-combine
 626	 */
 627	if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
 628	     new_pcm == _PAGE_CACHE_MODE_WB) ||
 629	    (pcm == _PAGE_CACHE_MODE_WC &&
 630	     new_pcm == _PAGE_CACHE_MODE_WB) ||
 631	    (pcm == _PAGE_CACHE_MODE_WT &&
 632	     new_pcm == _PAGE_CACHE_MODE_WB) ||
 633	    (pcm == _PAGE_CACHE_MODE_WT &&
 634	     new_pcm == _PAGE_CACHE_MODE_WC)) {
 635		return 0;
 636	}
 637
 638	return 1;
 639}
 640
 641pmd_t *populate_extra_pmd(unsigned long vaddr);
 642pte_t *populate_extra_pte(unsigned long vaddr);
 643#endif	/* __ASSEMBLY__ */
 644
 645#ifdef CONFIG_X86_32
 646# include <asm/pgtable_32.h>
 647#else
 648# include <asm/pgtable_64.h>
 649#endif
 650
 651#ifndef __ASSEMBLY__
 652#include <linux/mm_types.h>
 653#include <linux/mmdebug.h>
 654#include <linux/log2.h>
 655#include <asm/fixmap.h>
 656
 657static inline int pte_none(pte_t pte)
 658{
 659	return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK));
 660}
 661
 662#define __HAVE_ARCH_PTE_SAME
 663static inline int pte_same(pte_t a, pte_t b)
 664{
 665	return a.pte == b.pte;
 666}
 667
 668static inline int pte_present(pte_t a)
 669{
 670	return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
 671}
 672
 673#ifdef __HAVE_ARCH_PTE_DEVMAP
 674static inline int pte_devmap(pte_t a)
 675{
 676	return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP;
 677}
 678#endif
 679
 680#define pte_accessible pte_accessible
 681static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
 682{
 683	if (pte_flags(a) & _PAGE_PRESENT)
 684		return true;
 685
 686	if ((pte_flags(a) & _PAGE_PROTNONE) &&
 687			mm_tlb_flush_pending(mm))
 688		return true;
 689
 690	return false;
 691}
 692
 693static inline int pmd_present(pmd_t pmd)
 694{
 695	/*
 696	 * Checking for _PAGE_PSE is needed too because
 697	 * split_huge_page will temporarily clear the present bit (but
 698	 * the _PAGE_PSE flag will remain set at all times while the
 699	 * _PAGE_PRESENT bit is clear).
 700	 */
 701	return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
 702}
 703
 704#ifdef CONFIG_NUMA_BALANCING
 705/*
 706 * These work without NUMA balancing but the kernel does not care. See the
 707 * comment in include/asm-generic/pgtable.h
 708 */
 709static inline int pte_protnone(pte_t pte)
 710{
 711	return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
 712		== _PAGE_PROTNONE;
 713}
 714
 715static inline int pmd_protnone(pmd_t pmd)
 716{
 717	return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
 718		== _PAGE_PROTNONE;
 719}
 720#endif /* CONFIG_NUMA_BALANCING */
 721
 722static inline int pmd_none(pmd_t pmd)
 723{
 724	/* Only check low word on 32-bit platforms, since it might be
 725	   out of sync with upper half. */
 726	unsigned long val = native_pmd_val(pmd);
 727	return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0;
 728}
 729
 730static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 731{
 732	return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd));
 733}
 734
 735/*
 736 * Currently stuck as a macro due to indirect forward reference to
 737 * linux/mmzone.h's __section_mem_map_addr() definition:
 738 */
 739#define pmd_page(pmd)	pfn_to_page(pmd_pfn(pmd))
 740
 741/*
 742 * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
 743 *
 744 * this macro returns the index of the entry in the pmd page which would
 745 * control the given virtual address
 746 */
 747static inline unsigned long pmd_index(unsigned long address)
 748{
 749	return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
 750}
 751
 752/*
 753 * Conversion functions: convert a page and protection to a page entry,
 754 * and a page entry and page directory to the page they refer to.
 755 *
 756 * (Currently stuck as a macro because of indirect forward reference
 757 * to linux/mm.h:page_to_nid())
 758 */
 759#define mk_pte(page, pgprot)   pfn_pte(page_to_pfn(page), (pgprot))
 760
 761/*
 762 * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE]
 763 *
 764 * this function returns the index of the entry in the pte page which would
 765 * control the given virtual address
 766 */
 767static inline unsigned long pte_index(unsigned long address)
 768{
 769	return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
 770}
 771
 772static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
 773{
 774	return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
 775}
 776
 777static inline int pmd_bad(pmd_t pmd)
 778{
 779	return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
 780}
 781
 782static inline unsigned long pages_to_mb(unsigned long npg)
 783{
 784	return npg >> (20 - PAGE_SHIFT);
 785}
 786
 787#if CONFIG_PGTABLE_LEVELS > 2
 
 
 
 788static inline int pud_none(pud_t pud)
 789{
 790	return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
 791}
 792
 793static inline int pud_present(pud_t pud)
 794{
 795	return pud_flags(pud) & _PAGE_PRESENT;
 796}
 797
 798static inline unsigned long pud_page_vaddr(pud_t pud)
 799{
 800	return (unsigned long)__va(pud_val(pud) & pud_pfn_mask(pud));
 801}
 802
 803/*
 804 * Currently stuck as a macro due to indirect forward reference to
 805 * linux/mmzone.h's __section_mem_map_addr() definition:
 806 */
 807#define pud_page(pud)	pfn_to_page(pud_pfn(pud))
 808
 809/* Find an entry in the second-level page table.. */
 810static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
 811{
 812	return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
 813}
 814
 815static inline int pud_large(pud_t pud)
 816{
 817	return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
 818		(_PAGE_PSE | _PAGE_PRESENT);
 819}
 820
 821static inline int pud_bad(pud_t pud)
 822{
 823	return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
 824}
 825#else
 826static inline int pud_large(pud_t pud)
 827{
 828	return 0;
 829}
 830#endif	/* CONFIG_PGTABLE_LEVELS > 2 */
 831
 832static inline unsigned long pud_index(unsigned long address)
 833{
 834	return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
 835}
 836
 837#if CONFIG_PGTABLE_LEVELS > 3
 838static inline int p4d_none(p4d_t p4d)
 839{
 840	return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
 841}
 842
 843static inline int p4d_present(p4d_t p4d)
 844{
 845	return p4d_flags(p4d) & _PAGE_PRESENT;
 846}
 847
 848static inline unsigned long p4d_page_vaddr(p4d_t p4d)
 849{
 850	return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d));
 851}
 852
 853/*
 854 * Currently stuck as a macro due to indirect forward reference to
 855 * linux/mmzone.h's __section_mem_map_addr() definition:
 856 */
 857#define p4d_page(p4d)	pfn_to_page(p4d_pfn(p4d))
 858
 859/* Find an entry in the third-level page table.. */
 860static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
 861{
 862	return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address);
 863}
 864
 865static inline int p4d_bad(p4d_t p4d)
 866{
 867	unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER;
 868
 869	if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
 870		ignore_flags |= _PAGE_NX;
 871
 872	return (p4d_flags(p4d) & ~ignore_flags) != 0;
 873}
 874#endif  /* CONFIG_PGTABLE_LEVELS > 3 */
 875
 876static inline unsigned long p4d_index(unsigned long address)
 877{
 878	return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
 879}
 880
 881#if CONFIG_PGTABLE_LEVELS > 4
 882static inline int pgd_present(pgd_t pgd)
 883{
 884	if (!pgtable_l5_enabled)
 885		return 1;
 886	return pgd_flags(pgd) & _PAGE_PRESENT;
 887}
 888
 889static inline unsigned long pgd_page_vaddr(pgd_t pgd)
 890{
 891	return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
 892}
 893
 894/*
 895 * Currently stuck as a macro due to indirect forward reference to
 896 * linux/mmzone.h's __section_mem_map_addr() definition:
 897 */
 898#define pgd_page(pgd)	pfn_to_page(pgd_pfn(pgd))
 899
 900/* to find an entry in a page-table-directory. */
 901static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
 902{
 903	if (!pgtable_l5_enabled)
 904		return (p4d_t *)pgd;
 905	return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
 906}
 907
 908static inline int pgd_bad(pgd_t pgd)
 909{
 910	unsigned long ignore_flags = _PAGE_USER;
 911
 912	if (!pgtable_l5_enabled)
 913		return 0;
 914
 915	if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
 916		ignore_flags |= _PAGE_NX;
 917
 918	return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
 
 
 919}
 920
 921static inline int pgd_none(pgd_t pgd)
 922{
 923	if (!pgtable_l5_enabled)
 924		return 0;
 925	/*
 926	 * There is no need to do a workaround for the KNL stray
 927	 * A/D bit erratum here.  PGDs only point to page tables
 928	 * except on 32-bit non-PAE which is not supported on
 929	 * KNL.
 930	 */
 931	return !native_pgd_val(pgd);
 932}
 933#endif	/* CONFIG_PGTABLE_LEVELS > 4 */
 934
 935#endif	/* __ASSEMBLY__ */
 936
 937/*
 938 * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD]
 939 *
 940 * this macro returns the index of the entry in the pgd page which would
 941 * control the given virtual address
 942 */
 943#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
 944
 945/*
 946 * pgd_offset() returns a (pgd_t *)
 947 * pgd_index() is used get the offset into the pgd page's array of pgd_t's;
 948 */
 949#define pgd_offset_pgd(pgd, address) (pgd + pgd_index((address)))
 950/*
 951 * a shortcut to get a pgd_t in a given mm
 952 */
 953#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
 954/*
 955 * a shortcut which implies the use of the kernel's pgd, instead
 956 * of a process's
 957 */
 958#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
 959
 960
 961#define KERNEL_PGD_BOUNDARY	pgd_index(PAGE_OFFSET)
 962#define KERNEL_PGD_PTRS		(PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
 963
 964#ifndef __ASSEMBLY__
 965
 966extern int direct_gbpages;
 967void init_mem_mapping(void);
 968void early_alloc_pgt_buf(void);
 969extern void memblock_find_dma_reserve(void);
 970
 971#ifdef CONFIG_X86_64
 972/* Realmode trampoline initialization. */
 973extern pgd_t trampoline_pgd_entry;
 974static inline void __meminit init_trampoline_default(void)
 975{
 976	/* Default trampoline pgd value */
 977	trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
 978}
 979# ifdef CONFIG_RANDOMIZE_MEMORY
 980void __meminit init_trampoline(void);
 981# else
 982#  define init_trampoline init_trampoline_default
 983# endif
 984#else
 985static inline void init_trampoline(void) { }
 986#endif
 987
 988/* local pte updates need not use xchg for locking */
 989static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
 990{
 991	pte_t res = *ptep;
 992
 993	/* Pure native function needs no input for mm, addr */
 994	native_pte_clear(NULL, 0, ptep);
 995	return res;
 996}
 997
 998static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
 999{
1000	pmd_t res = *pmdp;
1001
1002	native_pmd_clear(pmdp);
1003	return res;
1004}
1005
1006static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp)
1007{
1008	pud_t res = *pudp;
1009
1010	native_pud_clear(pudp);
1011	return res;
1012}
1013
1014static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
1015				     pte_t *ptep , pte_t pte)
1016{
1017	native_set_pte(ptep, pte);
1018}
1019
1020static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1021			      pmd_t *pmdp, pmd_t pmd)
1022{
1023	native_set_pmd(pmdp, pmd);
1024}
1025
1026static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
1027			      pud_t *pudp, pud_t pud)
1028{
1029	native_set_pud(pudp, pud);
1030}
 
 
 
 
 
 
 
 
 
 
 
 
1031
1032/*
1033 * We only update the dirty/accessed state if we set
1034 * the dirty bit by hand in the kernel, since the hardware
1035 * will do the accessed bit for us, and we don't want to
1036 * race with other CPU's that might be updating the dirty
1037 * bit at the same time.
1038 */
1039struct vm_area_struct;
1040
1041#define  __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1042extern int ptep_set_access_flags(struct vm_area_struct *vma,
1043				 unsigned long address, pte_t *ptep,
1044				 pte_t entry, int dirty);
1045
1046#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1047extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
1048				     unsigned long addr, pte_t *ptep);
1049
1050#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1051extern int ptep_clear_flush_young(struct vm_area_struct *vma,
1052				  unsigned long address, pte_t *ptep);
1053
1054#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1055static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
1056				       pte_t *ptep)
1057{
1058	pte_t pte = native_ptep_get_and_clear(ptep);
 
1059	return pte;
1060}
1061
1062#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1063static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1064					    unsigned long addr, pte_t *ptep,
1065					    int full)
1066{
1067	pte_t pte;
1068	if (full) {
1069		/*
1070		 * Full address destruction in progress; paravirt does not
1071		 * care about updates and native needs no locking
1072		 */
1073		pte = native_local_ptep_get_and_clear(ptep);
1074	} else {
1075		pte = ptep_get_and_clear(mm, addr, ptep);
1076	}
1077	return pte;
1078}
1079
1080#define __HAVE_ARCH_PTEP_SET_WRPROTECT
1081static inline void ptep_set_wrprotect(struct mm_struct *mm,
1082				      unsigned long addr, pte_t *ptep)
1083{
1084	clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
 
1085}
1086
1087#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
1088
1089#define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))
1090
1091#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1092extern int pmdp_set_access_flags(struct vm_area_struct *vma,
1093				 unsigned long address, pmd_t *pmdp,
1094				 pmd_t entry, int dirty);
1095extern int pudp_set_access_flags(struct vm_area_struct *vma,
1096				 unsigned long address, pud_t *pudp,
1097				 pud_t entry, int dirty);
1098
1099#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1100extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1101				     unsigned long addr, pmd_t *pmdp);
1102extern int pudp_test_and_clear_young(struct vm_area_struct *vma,
1103				     unsigned long addr, pud_t *pudp);
1104
1105#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
1106extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
1107				  unsigned long address, pmd_t *pmdp);
1108
1109
1110#define pmd_write pmd_write
 
 
 
 
1111static inline int pmd_write(pmd_t pmd)
1112{
1113	return pmd_flags(pmd) & _PAGE_RW;
1114}
1115
1116#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1117static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
1118				       pmd_t *pmdp)
1119{
1120	return native_pmdp_get_and_clear(pmdp);
1121}
1122
1123#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
1124static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
1125					unsigned long addr, pud_t *pudp)
1126{
1127	return native_pudp_get_and_clear(pudp);
1128}
1129
1130#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1131static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1132				      unsigned long addr, pmd_t *pmdp)
1133{
1134	clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
 
1135}
1136
1137#define pud_write pud_write
1138static inline int pud_write(pud_t pud)
1139{
1140	return pud_flags(pud) & _PAGE_RW;
1141}
1142
1143#ifndef pmdp_establish
1144#define pmdp_establish pmdp_establish
1145static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1146		unsigned long address, pmd_t *pmdp, pmd_t pmd)
1147{
1148	if (IS_ENABLED(CONFIG_SMP)) {
1149		return xchg(pmdp, pmd);
1150	} else {
1151		pmd_t old = *pmdp;
1152		*pmdp = pmd;
1153		return old;
1154	}
1155}
1156#endif
1157
1158/*
1159 * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
1160 *
1161 *  dst - pointer to pgd range anwhere on a pgd page
1162 *  src - ""
1163 *  count - the number of pgds to copy.
1164 *
1165 * dst and src can be on the same page, but the range must not overlap,
1166 * and must not cross a page boundary.
1167 */
1168static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
1169{
1170	memcpy(dst, src, count * sizeof(pgd_t));
1171#ifdef CONFIG_PAGE_TABLE_ISOLATION
1172	if (!static_cpu_has(X86_FEATURE_PTI))
1173		return;
1174	/* Clone the user space pgd as well */
1175	memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
1176	       count * sizeof(pgd_t));
1177#endif
1178}
1179
1180#define PTE_SHIFT ilog2(PTRS_PER_PTE)
1181static inline int page_level_shift(enum pg_level level)
1182{
1183	return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
1184}
1185static inline unsigned long page_level_size(enum pg_level level)
1186{
1187	return 1UL << page_level_shift(level);
1188}
1189static inline unsigned long page_level_mask(enum pg_level level)
1190{
1191	return ~(page_level_size(level) - 1);
1192}
1193
1194/*
1195 * The x86 doesn't have any external MMU info: the kernel page
1196 * tables contain all the necessary information.
1197 */
1198static inline void update_mmu_cache(struct vm_area_struct *vma,
1199		unsigned long addr, pte_t *ptep)
1200{
1201}
1202static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
1203		unsigned long addr, pmd_t *pmd)
1204{
1205}
1206static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
1207		unsigned long addr, pud_t *pud)
1208{
1209}
1210
1211#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1212static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
1213{
1214	return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1215}
1216
1217static inline int pte_swp_soft_dirty(pte_t pte)
1218{
1219	return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
1220}
1221
1222static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
1223{
1224	return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1225}
1226
1227#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1228static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1229{
1230	return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1231}
1232
1233static inline int pmd_swp_soft_dirty(pmd_t pmd)
1234{
1235	return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY;
1236}
1237
1238static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1239{
1240	return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1241}
1242#endif
1243#endif
1244
1245#define PKRU_AD_BIT 0x1
1246#define PKRU_WD_BIT 0x2
1247#define PKRU_BITS_PER_PKEY 2
1248
1249static inline bool __pkru_allows_read(u32 pkru, u16 pkey)
1250{
1251	int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
1252	return !(pkru & (PKRU_AD_BIT << pkru_pkey_bits));
1253}
1254
1255static inline bool __pkru_allows_write(u32 pkru, u16 pkey)
1256{
1257	int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
1258	/*
1259	 * Access-disable disables writes too so we need to check
1260	 * both bits here.
1261	 */
1262	return !(pkru & ((PKRU_AD_BIT|PKRU_WD_BIT) << pkru_pkey_bits));
1263}
1264
1265static inline u16 pte_flags_pkey(unsigned long pte_flags)
1266{
1267#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1268	/* ifdef to avoid doing 59-bit shift on 32-bit values */
1269	return (pte_flags & _PAGE_PKEY_MASK) >> _PAGE_BIT_PKEY_BIT0;
1270#else
1271	return 0;
1272#endif
1273}
1274
1275static inline bool __pkru_allows_pkey(u16 pkey, bool write)
1276{
1277	u32 pkru = read_pkru();
1278
1279	if (!__pkru_allows_read(pkru, pkey))
1280		return false;
1281	if (write && !__pkru_allows_write(pkru, pkey))
1282		return false;
1283
1284	return true;
1285}
1286
1287/*
1288 * 'pteval' can come from a PTE, PMD or PUD.  We only check
1289 * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the
1290 * same value on all 3 types.
1291 */
1292static inline bool __pte_access_permitted(unsigned long pteval, bool write)
1293{
1294	unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
1295
1296	if (write)
1297		need_pte_bits |= _PAGE_RW;
1298
1299	if ((pteval & need_pte_bits) != need_pte_bits)
1300		return 0;
1301
1302	return __pkru_allows_pkey(pte_flags_pkey(pteval), write);
1303}
1304
1305#define pte_access_permitted pte_access_permitted
1306static inline bool pte_access_permitted(pte_t pte, bool write)
1307{
1308	return __pte_access_permitted(pte_val(pte), write);
1309}
1310
1311#define pmd_access_permitted pmd_access_permitted
1312static inline bool pmd_access_permitted(pmd_t pmd, bool write)
1313{
1314	return __pte_access_permitted(pmd_val(pmd), write);
1315}
1316
1317#define pud_access_permitted pud_access_permitted
1318static inline bool pud_access_permitted(pud_t pud, bool write)
1319{
1320	return __pte_access_permitted(pud_val(pud), write);
1321}
1322
1323#include <asm-generic/pgtable.h>
1324#endif	/* __ASSEMBLY__ */
1325
1326#endif /* _ASM_X86_PGTABLE_H */