Loading...
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 2012 ARM Limited
3// Copyright (C) 2005-2017 Andes Technology Corporation
4
5#include <linux/cache.h>
6#include <linux/clocksource.h>
7#include <linux/elf.h>
8#include <linux/err.h>
9#include <linux/errno.h>
10#include <linux/gfp.h>
11#include <linux/kernel.h>
12#include <linux/mm.h>
13#include <linux/sched.h>
14#include <linux/signal.h>
15#include <linux/slab.h>
16#include <linux/timekeeper_internal.h>
17#include <linux/vmalloc.h>
18#include <linux/random.h>
19
20#include <asm/cacheflush.h>
21#include <asm/vdso.h>
22#include <asm/vdso_datapage.h>
23#include <asm/vdso_timer_info.h>
24#include <asm/cache_info.h>
25extern struct cache_info L1_cache_info[2];
26extern char vdso_start[], vdso_end[];
27static unsigned long vdso_pages __ro_after_init;
28static unsigned long timer_mapping_base;
29
30struct timer_info_t timer_info = {
31 .cycle_count_down = true,
32 .mapping_base = EMPTY_TIMER_MAPPING,
33 .cycle_count_reg_offset = EMPTY_REG_OFFSET
34};
35/*
36 * The vDSO data page.
37 */
38static struct page *no_pages[] = { NULL };
39
40static union {
41 struct vdso_data data;
42 u8 page[PAGE_SIZE];
43} vdso_data_store __page_aligned_data;
44struct vdso_data *vdso_data = &vdso_data_store.data;
45static struct vm_special_mapping vdso_spec[2] __ro_after_init = {
46 {
47 .name = "[vvar]",
48 .pages = no_pages,
49 },
50 {
51 .name = "[vdso]",
52 },
53};
54
55static void get_timer_node_info(void)
56{
57 timer_mapping_base = timer_info.mapping_base;
58 vdso_data->cycle_count_offset =
59 timer_info.cycle_count_reg_offset;
60 vdso_data->cycle_count_down =
61 timer_info.cycle_count_down;
62}
63
64static int __init vdso_init(void)
65{
66 int i;
67 struct page **vdso_pagelist;
68
69 if (memcmp(vdso_start, "\177ELF", 4)) {
70 pr_err("vDSO is not a valid ELF object!\n");
71 return -EINVAL;
72 }
73 /* Creat a timer io mapping to get clock cycles counter */
74 get_timer_node_info();
75
76 vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT;
77 pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n",
78 vdso_pages + 1, vdso_pages, vdso_start, 1L, vdso_data);
79
80 /* Allocate the vDSO pagelist */
81 vdso_pagelist = kcalloc(vdso_pages, sizeof(struct page *), GFP_KERNEL);
82 if (vdso_pagelist == NULL)
83 return -ENOMEM;
84
85 for (i = 0; i < vdso_pages; i++)
86 vdso_pagelist[i] = virt_to_page(vdso_start + i * PAGE_SIZE);
87 vdso_spec[1].pages = &vdso_pagelist[0];
88
89 return 0;
90}
91
92arch_initcall(vdso_init);
93
94unsigned long inline vdso_random_addr(unsigned long vdso_mapping_len)
95{
96 unsigned long start = current->mm->mmap_base, end, offset, addr;
97 start = PAGE_ALIGN(start);
98
99 /* Round the lowest possible end address up to a PMD boundary. */
100 end = (start + vdso_mapping_len + PMD_SIZE - 1) & PMD_MASK;
101 if (end >= TASK_SIZE)
102 end = TASK_SIZE;
103 end -= vdso_mapping_len;
104
105 if (end > start) {
106 offset = get_random_int() % (((end - start) >> PAGE_SHIFT) + 1);
107 addr = start + (offset << PAGE_SHIFT);
108 } else {
109 addr = start;
110 }
111 return addr;
112}
113
114int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
115{
116 struct mm_struct *mm = current->mm;
117 unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
118 struct vm_area_struct *vma;
119 unsigned long addr = 0;
120 pgprot_t prot;
121 int ret, vvar_page_num = 2;
122
123 vdso_text_len = vdso_pages << PAGE_SHIFT;
124
125 if(timer_mapping_base == EMPTY_VALUE)
126 vvar_page_num = 1;
127 /* Be sure to map the data page */
128 vdso_mapping_len = vdso_text_len + vvar_page_num * PAGE_SIZE;
129#ifdef CONFIG_CPU_CACHE_ALIASING
130 vdso_mapping_len += L1_cache_info[DCACHE].aliasing_num - 1;
131#endif
132
133 if (down_write_killable(&mm->mmap_sem))
134 return -EINTR;
135
136 addr = vdso_random_addr(vdso_mapping_len);
137 vdso_base = get_unmapped_area(NULL, addr, vdso_mapping_len, 0, 0);
138 if (IS_ERR_VALUE(vdso_base)) {
139 ret = vdso_base;
140 goto up_fail;
141 }
142
143#ifdef CONFIG_CPU_CACHE_ALIASING
144 {
145 unsigned int aliasing_mask =
146 L1_cache_info[DCACHE].aliasing_mask;
147 unsigned int page_colour_ofs;
148 page_colour_ofs = ((unsigned int)vdso_data & aliasing_mask) -
149 (vdso_base & aliasing_mask);
150 vdso_base += page_colour_ofs & aliasing_mask;
151 }
152#endif
153
154 vma = _install_special_mapping(mm, vdso_base, vvar_page_num * PAGE_SIZE,
155 VM_READ | VM_MAYREAD, &vdso_spec[0]);
156 if (IS_ERR(vma)) {
157 ret = PTR_ERR(vma);
158 goto up_fail;
159 }
160
161 /*Map vdata to user space */
162 ret = io_remap_pfn_range(vma, vdso_base,
163 virt_to_phys(vdso_data) >> PAGE_SHIFT,
164 PAGE_SIZE, vma->vm_page_prot);
165 if (ret)
166 goto up_fail;
167
168 /*Map timer to user space */
169 vdso_base += PAGE_SIZE;
170 prot = __pgprot(_PAGE_V | _PAGE_M_UR_KR | _PAGE_D | _PAGE_C_DEV);
171 ret = io_remap_pfn_range(vma, vdso_base, timer_mapping_base >> PAGE_SHIFT,
172 PAGE_SIZE, prot);
173 if (ret)
174 goto up_fail;
175
176 /*Map vdso to user space */
177 vdso_base += PAGE_SIZE;
178 mm->context.vdso = (void *)vdso_base;
179 vma = _install_special_mapping(mm, vdso_base, vdso_text_len,
180 VM_READ | VM_EXEC |
181 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC,
182 &vdso_spec[1]);
183 if (IS_ERR(vma)) {
184 ret = PTR_ERR(vma);
185 goto up_fail;
186 }
187
188 up_write(&mm->mmap_sem);
189 return 0;
190
191up_fail:
192 mm->context.vdso = NULL;
193 up_write(&mm->mmap_sem);
194 return ret;
195}
196
197static void vdso_write_begin(struct vdso_data *vdata)
198{
199 ++vdso_data->seq_count;
200 smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */
201}
202
203static void vdso_write_end(struct vdso_data *vdata)
204{
205 smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */
206 ++vdso_data->seq_count;
207}
208
209void update_vsyscall(struct timekeeper *tk)
210{
211 vdso_write_begin(vdso_data);
212 vdso_data->cs_mask = tk->tkr_mono.mask;
213 vdso_data->cs_mult = tk->tkr_mono.mult;
214 vdso_data->cs_shift = tk->tkr_mono.shift;
215 vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
216 vdso_data->wtm_clock_sec = tk->wall_to_monotonic.tv_sec;
217 vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec;
218 vdso_data->xtime_clock_sec = tk->xtime_sec;
219 vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec;
220 vdso_data->xtime_coarse_sec = tk->xtime_sec;
221 vdso_data->xtime_coarse_nsec = tk->tkr_mono.xtime_nsec >>
222 tk->tkr_mono.shift;
223 vdso_data->hrtimer_res = hrtimer_resolution;
224 vdso_write_end(vdso_data);
225}
226
227void update_vsyscall_tz(void)
228{
229 vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;
230 vdso_data->tz_dsttime = sys_tz.tz_dsttime;
231}
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 2012 ARM Limited
3// Copyright (C) 2005-2017 Andes Technology Corporation
4
5#include <linux/cache.h>
6#include <linux/clocksource.h>
7#include <linux/elf.h>
8#include <linux/err.h>
9#include <linux/errno.h>
10#include <linux/gfp.h>
11#include <linux/kernel.h>
12#include <linux/mm.h>
13#include <linux/sched.h>
14#include <linux/signal.h>
15#include <linux/slab.h>
16#include <linux/timekeeper_internal.h>
17#include <linux/vmalloc.h>
18#include <linux/random.h>
19
20#include <asm/cacheflush.h>
21#include <asm/vdso.h>
22#include <asm/vdso_datapage.h>
23#include <asm/vdso_timer_info.h>
24#include <asm/cache_info.h>
25extern struct cache_info L1_cache_info[2];
26extern char vdso_start[], vdso_end[];
27static unsigned long vdso_pages __ro_after_init;
28static unsigned long timer_mapping_base;
29
30struct timer_info_t timer_info = {
31 .cycle_count_down = true,
32 .mapping_base = EMPTY_TIMER_MAPPING,
33 .cycle_count_reg_offset = EMPTY_REG_OFFSET
34};
35/*
36 * The vDSO data page.
37 */
38static struct page *no_pages[] = { NULL };
39
40static union {
41 struct vdso_data data;
42 u8 page[PAGE_SIZE];
43} vdso_data_store __page_aligned_data;
44struct vdso_data *vdso_data = &vdso_data_store.data;
45static struct vm_special_mapping vdso_spec[2] __ro_after_init = {
46 {
47 .name = "[vvar]",
48 .pages = no_pages,
49 },
50 {
51 .name = "[vdso]",
52 },
53};
54
55static void get_timer_node_info(void)
56{
57 timer_mapping_base = timer_info.mapping_base;
58 vdso_data->cycle_count_offset =
59 timer_info.cycle_count_reg_offset;
60 vdso_data->cycle_count_down =
61 timer_info.cycle_count_down;
62}
63
64static int __init vdso_init(void)
65{
66 int i;
67 struct page **vdso_pagelist;
68
69 if (memcmp(vdso_start, "\177ELF", 4)) {
70 pr_err("vDSO is not a valid ELF object!\n");
71 return -EINVAL;
72 }
73 /* Creat a timer io mapping to get clock cycles counter */
74 get_timer_node_info();
75
76 vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT;
77 pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n",
78 vdso_pages + 1, vdso_pages, vdso_start, 1L, vdso_data);
79
80 /* Allocate the vDSO pagelist */
81 vdso_pagelist = kcalloc(vdso_pages, sizeof(struct page *), GFP_KERNEL);
82 if (vdso_pagelist == NULL)
83 return -ENOMEM;
84
85 for (i = 0; i < vdso_pages; i++)
86 vdso_pagelist[i] = virt_to_page(vdso_start + i * PAGE_SIZE);
87 vdso_spec[1].pages = &vdso_pagelist[0];
88
89 return 0;
90}
91
92arch_initcall(vdso_init);
93
94unsigned long inline vdso_random_addr(unsigned long vdso_mapping_len)
95{
96 unsigned long start = current->mm->mmap_base, end, offset, addr;
97 start = PAGE_ALIGN(start);
98
99 /* Round the lowest possible end address up to a PMD boundary. */
100 end = (start + vdso_mapping_len + PMD_SIZE - 1) & PMD_MASK;
101 if (end >= TASK_SIZE)
102 end = TASK_SIZE;
103 end -= vdso_mapping_len;
104
105 if (end > start) {
106 offset = get_random_int() % (((end - start) >> PAGE_SHIFT) + 1);
107 addr = start + (offset << PAGE_SHIFT);
108 } else {
109 addr = start;
110 }
111 return addr;
112}
113
114int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
115{
116 struct mm_struct *mm = current->mm;
117 unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
118 struct vm_area_struct *vma;
119 unsigned long addr = 0;
120 pgprot_t prot;
121 int ret, vvar_page_num = 2;
122
123 vdso_text_len = vdso_pages << PAGE_SHIFT;
124
125 if(timer_mapping_base == EMPTY_VALUE)
126 vvar_page_num = 1;
127 /* Be sure to map the data page */
128 vdso_mapping_len = vdso_text_len + vvar_page_num * PAGE_SIZE;
129#ifdef CONFIG_CPU_CACHE_ALIASING
130 vdso_mapping_len += L1_cache_info[DCACHE].aliasing_num - 1;
131#endif
132
133 if (down_write_killable(&mm->mmap_sem))
134 return -EINTR;
135
136 addr = vdso_random_addr(vdso_mapping_len);
137 vdso_base = get_unmapped_area(NULL, addr, vdso_mapping_len, 0, 0);
138 if (IS_ERR_VALUE(vdso_base)) {
139 ret = vdso_base;
140 goto up_fail;
141 }
142
143#ifdef CONFIG_CPU_CACHE_ALIASING
144 {
145 unsigned int aliasing_mask =
146 L1_cache_info[DCACHE].aliasing_mask;
147 unsigned int page_colour_ofs;
148 page_colour_ofs = ((unsigned int)vdso_data & aliasing_mask) -
149 (vdso_base & aliasing_mask);
150 vdso_base += page_colour_ofs & aliasing_mask;
151 }
152#endif
153
154 vma = _install_special_mapping(mm, vdso_base, vvar_page_num * PAGE_SIZE,
155 VM_READ | VM_MAYREAD, &vdso_spec[0]);
156 if (IS_ERR(vma)) {
157 ret = PTR_ERR(vma);
158 goto up_fail;
159 }
160
161 /*Map vdata to user space */
162 ret = io_remap_pfn_range(vma, vdso_base,
163 virt_to_phys(vdso_data) >> PAGE_SHIFT,
164 PAGE_SIZE, vma->vm_page_prot);
165 if (ret)
166 goto up_fail;
167
168 /*Map timer to user space */
169 vdso_base += PAGE_SIZE;
170 prot = __pgprot(_PAGE_V | _PAGE_M_UR_KR | _PAGE_D | _PAGE_C_DEV);
171 ret = io_remap_pfn_range(vma, vdso_base, timer_mapping_base >> PAGE_SHIFT,
172 PAGE_SIZE, prot);
173 if (ret)
174 goto up_fail;
175
176 /*Map vdso to user space */
177 vdso_base += PAGE_SIZE;
178 mm->context.vdso = (void *)vdso_base;
179 vma = _install_special_mapping(mm, vdso_base, vdso_text_len,
180 VM_READ | VM_EXEC |
181 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC,
182 &vdso_spec[1]);
183 if (IS_ERR(vma)) {
184 ret = PTR_ERR(vma);
185 goto up_fail;
186 }
187
188 up_write(&mm->mmap_sem);
189 return 0;
190
191up_fail:
192 mm->context.vdso = NULL;
193 up_write(&mm->mmap_sem);
194 return ret;
195}
196
197static void vdso_write_begin(struct vdso_data *vdata)
198{
199 ++vdso_data->seq_count;
200 smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */
201}
202
203static void vdso_write_end(struct vdso_data *vdata)
204{
205 smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */
206 ++vdso_data->seq_count;
207}
208
209void update_vsyscall(struct timekeeper *tk)
210{
211 vdso_write_begin(vdso_data);
212 vdso_data->cs_mask = tk->tkr_mono.mask;
213 vdso_data->cs_mult = tk->tkr_mono.mult;
214 vdso_data->cs_shift = tk->tkr_mono.shift;
215 vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
216 vdso_data->wtm_clock_sec = tk->wall_to_monotonic.tv_sec;
217 vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec;
218 vdso_data->xtime_clock_sec = tk->xtime_sec;
219 vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec;
220 vdso_data->xtime_coarse_sec = tk->xtime_sec;
221 vdso_data->xtime_coarse_nsec = tk->tkr_mono.xtime_nsec >>
222 tk->tkr_mono.shift;
223 vdso_write_end(vdso_data);
224}
225
226void update_vsyscall_tz(void)
227{
228 vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;
229 vdso_data->tz_dsttime = sys_tz.tz_dsttime;
230}