Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Access to PCI I/O memory from user space programs.
4 *
5 * Copyright IBM Corp. 2014
6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
7 */
8#include <linux/kernel.h>
9#include <linux/syscalls.h>
10#include <linux/init.h>
11#include <linux/mm.h>
12#include <linux/errno.h>
13#include <linux/pci.h>
14#include <asm/asm-extable.h>
15#include <asm/pci_io.h>
16#include <asm/pci_debug.h>
17
18static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
19{
20 struct {
21 u64 offset;
22 u8 cc;
23 u8 status;
24 } data = {offset, cc, status};
25
26 zpci_err_hex(&data, sizeof(data));
27}
28
29static inline int __pcistb_mio_inuser(
30 void __iomem *ioaddr, const void __user *src,
31 u64 len, u8 *status)
32{
33 int cc = -ENXIO;
34
35 asm volatile (
36 " sacf 256\n"
37 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
38 "1: ipm %[cc]\n"
39 " srl %[cc],28\n"
40 "2: sacf 768\n"
41 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
42 : [cc] "+d" (cc), [len] "+d" (len)
43 : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
44 : "cc", "memory");
45 *status = len >> 24 & 0xff;
46 return cc;
47}
48
49static inline int __pcistg_mio_inuser(
50 void __iomem *ioaddr, const void __user *src,
51 u64 ulen, u8 *status)
52{
53 union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
54 int cc = -ENXIO;
55 u64 val = 0;
56 u64 cnt = ulen;
57 u8 tmp;
58
59 /*
60 * copy 0 < @len <= 8 bytes from @src into the right most bytes of
61 * a register, then store it to PCI at @ioaddr while in secondary
62 * address space. pcistg then uses the user mappings.
63 */
64 asm volatile (
65 " sacf 256\n"
66 "0: llgc %[tmp],0(%[src])\n"
67 "4: sllg %[val],%[val],8\n"
68 " aghi %[src],1\n"
69 " ogr %[val],%[tmp]\n"
70 " brctg %[cnt],0b\n"
71 "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n"
72 "2: ipm %[cc]\n"
73 " srl %[cc],28\n"
74 "3: sacf 768\n"
75 EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
76 :
77 [src] "+a" (src), [cnt] "+d" (cnt),
78 [val] "+d" (val), [tmp] "=d" (tmp),
79 [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair)
80 :: "cc", "memory");
81 *status = ioaddr_len.odd >> 24 & 0xff;
82
83 /* did we read everything from user memory? */
84 if (!cc && cnt != 0)
85 cc = -EFAULT;
86
87 return cc;
88}
89
90static inline int __memcpy_toio_inuser(void __iomem *dst,
91 const void __user *src, size_t n)
92{
93 int size, rc = 0;
94 u8 status = 0;
95
96 if (!src)
97 return -EINVAL;
98
99 while (n > 0) {
100 size = zpci_get_max_io_size((u64 __force) dst,
101 (u64 __force) src, n,
102 ZPCI_MAX_WRITE_SIZE);
103 if (size > 8) /* main path */
104 rc = __pcistb_mio_inuser(dst, src, size, &status);
105 else
106 rc = __pcistg_mio_inuser(dst, src, size, &status);
107 if (rc)
108 break;
109 src += size;
110 dst += size;
111 n -= size;
112 }
113 if (rc)
114 zpci_err_mmio(rc, status, (__force u64) dst);
115 return rc;
116}
117
118SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
119 const void __user *, user_buffer, size_t, length)
120{
121 u8 local_buf[64];
122 void __iomem *io_addr;
123 void *buf;
124 struct vm_area_struct *vma;
125 pte_t *ptep;
126 spinlock_t *ptl;
127 long ret;
128
129 if (!zpci_is_enabled())
130 return -ENODEV;
131
132 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
133 return -EINVAL;
134
135 /*
136 * We only support write access to MIO capable devices if we are on
137 * a MIO enabled system. Otherwise we would have to check for every
138 * address if it is a special ZPCI_ADDR and would have to do
139 * a pfn lookup which we don't need for MIO capable devices. Currently
140 * ISM devices are the only devices without MIO support and there is no
141 * known need for accessing these from userspace.
142 */
143 if (static_branch_likely(&have_mio)) {
144 ret = __memcpy_toio_inuser((void __iomem *) mmio_addr,
145 user_buffer,
146 length);
147 return ret;
148 }
149
150 if (length > 64) {
151 buf = kmalloc(length, GFP_KERNEL);
152 if (!buf)
153 return -ENOMEM;
154 } else
155 buf = local_buf;
156
157 ret = -EFAULT;
158 if (copy_from_user(buf, user_buffer, length))
159 goto out_free;
160
161 mmap_read_lock(current->mm);
162 ret = -EINVAL;
163 vma = vma_lookup(current->mm, mmio_addr);
164 if (!vma)
165 goto out_unlock_mmap;
166 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
167 goto out_unlock_mmap;
168 ret = -EACCES;
169 if (!(vma->vm_flags & VM_WRITE))
170 goto out_unlock_mmap;
171
172 ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
173 if (ret)
174 goto out_unlock_mmap;
175
176 io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
177 (mmio_addr & ~PAGE_MASK));
178
179 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
180 goto out_unlock_pt;
181
182 ret = zpci_memcpy_toio(io_addr, buf, length);
183out_unlock_pt:
184 pte_unmap_unlock(ptep, ptl);
185out_unlock_mmap:
186 mmap_read_unlock(current->mm);
187out_free:
188 if (buf != local_buf)
189 kfree(buf);
190 return ret;
191}
192
193static inline int __pcilg_mio_inuser(
194 void __user *dst, const void __iomem *ioaddr,
195 u64 ulen, u8 *status)
196{
197 union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
198 u64 cnt = ulen;
199 int shift = ulen * 8;
200 int cc = -ENXIO;
201 u64 val, tmp;
202
203 /*
204 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
205 * user space) into a register using pcilg then store these bytes at
206 * user address @dst
207 */
208 asm volatile (
209 " sacf 256\n"
210 "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n"
211 "1: ipm %[cc]\n"
212 " srl %[cc],28\n"
213 " ltr %[cc],%[cc]\n"
214 " jne 4f\n"
215 "2: ahi %[shift],-8\n"
216 " srlg %[tmp],%[val],0(%[shift])\n"
217 "3: stc %[tmp],0(%[dst])\n"
218 "5: aghi %[dst],1\n"
219 " brctg %[cnt],2b\n"
220 "4: sacf 768\n"
221 EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b)
222 :
223 [ioaddr_len] "+&d" (ioaddr_len.pair),
224 [cc] "+d" (cc), [val] "=d" (val),
225 [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
226 [shift] "+d" (shift)
227 :: "cc", "memory");
228
229 /* did we write everything to the user space buffer? */
230 if (!cc && cnt != 0)
231 cc = -EFAULT;
232
233 *status = ioaddr_len.odd >> 24 & 0xff;
234 return cc;
235}
236
237static inline int __memcpy_fromio_inuser(void __user *dst,
238 const void __iomem *src,
239 unsigned long n)
240{
241 int size, rc = 0;
242 u8 status;
243
244 while (n > 0) {
245 size = zpci_get_max_io_size((u64 __force) src,
246 (u64 __force) dst, n,
247 ZPCI_MAX_READ_SIZE);
248 rc = __pcilg_mio_inuser(dst, src, size, &status);
249 if (rc)
250 break;
251 src += size;
252 dst += size;
253 n -= size;
254 }
255 if (rc)
256 zpci_err_mmio(rc, status, (__force u64) dst);
257 return rc;
258}
259
260SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
261 void __user *, user_buffer, size_t, length)
262{
263 u8 local_buf[64];
264 void __iomem *io_addr;
265 void *buf;
266 struct vm_area_struct *vma;
267 pte_t *ptep;
268 spinlock_t *ptl;
269 long ret;
270
271 if (!zpci_is_enabled())
272 return -ENODEV;
273
274 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
275 return -EINVAL;
276
277 /*
278 * We only support read access to MIO capable devices if we are on
279 * a MIO enabled system. Otherwise we would have to check for every
280 * address if it is a special ZPCI_ADDR and would have to do
281 * a pfn lookup which we don't need for MIO capable devices. Currently
282 * ISM devices are the only devices without MIO support and there is no
283 * known need for accessing these from userspace.
284 */
285 if (static_branch_likely(&have_mio)) {
286 ret = __memcpy_fromio_inuser(
287 user_buffer, (const void __iomem *)mmio_addr,
288 length);
289 return ret;
290 }
291
292 if (length > 64) {
293 buf = kmalloc(length, GFP_KERNEL);
294 if (!buf)
295 return -ENOMEM;
296 } else {
297 buf = local_buf;
298 }
299
300 mmap_read_lock(current->mm);
301 ret = -EINVAL;
302 vma = vma_lookup(current->mm, mmio_addr);
303 if (!vma)
304 goto out_unlock_mmap;
305 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
306 goto out_unlock_mmap;
307 ret = -EACCES;
308 if (!(vma->vm_flags & VM_WRITE))
309 goto out_unlock_mmap;
310
311 ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
312 if (ret)
313 goto out_unlock_mmap;
314
315 io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
316 (mmio_addr & ~PAGE_MASK));
317
318 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
319 ret = -EFAULT;
320 goto out_unlock_pt;
321 }
322 ret = zpci_memcpy_fromio(buf, io_addr, length);
323
324out_unlock_pt:
325 pte_unmap_unlock(ptep, ptl);
326out_unlock_mmap:
327 mmap_read_unlock(current->mm);
328
329 if (!ret && copy_to_user(user_buffer, buf, length))
330 ret = -EFAULT;
331
332 if (buf != local_buf)
333 kfree(buf);
334 return ret;
335}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Access to PCI I/O memory from user space programs.
4 *
5 * Copyright IBM Corp. 2014
6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
7 */
8#include <linux/kernel.h>
9#include <linux/syscalls.h>
10#include <linux/init.h>
11#include <linux/mm.h>
12#include <linux/errno.h>
13#include <linux/pci.h>
14#include <asm/pci_io.h>
15#include <asm/pci_debug.h>
16
17static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
18{
19 struct {
20 u64 offset;
21 u8 cc;
22 u8 status;
23 } data = {offset, cc, status};
24
25 zpci_err_hex(&data, sizeof(data));
26}
27
28static inline int __pcistb_mio_inuser(
29 void __iomem *ioaddr, const void __user *src,
30 u64 len, u8 *status)
31{
32 int cc = -ENXIO;
33
34 asm volatile (
35 " sacf 256\n"
36 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
37 "1: ipm %[cc]\n"
38 " srl %[cc],28\n"
39 "2: sacf 768\n"
40 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
41 : [cc] "+d" (cc), [len] "+d" (len)
42 : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
43 : "cc", "memory");
44 *status = len >> 24 & 0xff;
45 return cc;
46}
47
48static inline int __pcistg_mio_inuser(
49 void __iomem *ioaddr, const void __user *src,
50 u64 ulen, u8 *status)
51{
52 register u64 addr asm("2") = (u64 __force) ioaddr;
53 register u64 len asm("3") = ulen;
54 int cc = -ENXIO;
55 u64 val = 0;
56 u64 cnt = ulen;
57 u8 tmp;
58
59 /*
60 * copy 0 < @len <= 8 bytes from @src into the right most bytes of
61 * a register, then store it to PCI at @ioaddr while in secondary
62 * address space. pcistg then uses the user mappings.
63 */
64 asm volatile (
65 " sacf 256\n"
66 "0: llgc %[tmp],0(%[src])\n"
67 " sllg %[val],%[val],8\n"
68 " aghi %[src],1\n"
69 " ogr %[val],%[tmp]\n"
70 " brctg %[cnt],0b\n"
71 "1: .insn rre,0xb9d40000,%[val],%[ioaddr]\n"
72 "2: ipm %[cc]\n"
73 " srl %[cc],28\n"
74 "3: sacf 768\n"
75 EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
76 :
77 [src] "+a" (src), [cnt] "+d" (cnt),
78 [val] "+d" (val), [tmp] "=d" (tmp),
79 [len] "+d" (len), [cc] "+d" (cc),
80 [ioaddr] "+a" (addr)
81 :: "cc", "memory");
82 *status = len >> 24 & 0xff;
83
84 /* did we read everything from user memory? */
85 if (!cc && cnt != 0)
86 cc = -EFAULT;
87
88 return cc;
89}
90
91static inline int __memcpy_toio_inuser(void __iomem *dst,
92 const void __user *src, size_t n)
93{
94 int size, rc = 0;
95 u8 status = 0;
96 mm_segment_t old_fs;
97
98 if (!src)
99 return -EINVAL;
100
101 old_fs = enable_sacf_uaccess();
102 while (n > 0) {
103 size = zpci_get_max_write_size((u64 __force) dst,
104 (u64 __force) src, n,
105 ZPCI_MAX_WRITE_SIZE);
106 if (size > 8) /* main path */
107 rc = __pcistb_mio_inuser(dst, src, size, &status);
108 else
109 rc = __pcistg_mio_inuser(dst, src, size, &status);
110 if (rc)
111 break;
112 src += size;
113 dst += size;
114 n -= size;
115 }
116 disable_sacf_uaccess(old_fs);
117 if (rc)
118 zpci_err_mmio(rc, status, (__force u64) dst);
119 return rc;
120}
121
122static long get_pfn(unsigned long user_addr, unsigned long access,
123 unsigned long *pfn)
124{
125 struct vm_area_struct *vma;
126 long ret;
127
128 mmap_read_lock(current->mm);
129 ret = -EINVAL;
130 vma = find_vma(current->mm, user_addr);
131 if (!vma)
132 goto out;
133 ret = -EACCES;
134 if (!(vma->vm_flags & access))
135 goto out;
136 ret = follow_pfn(vma, user_addr, pfn);
137out:
138 mmap_read_unlock(current->mm);
139 return ret;
140}
141
142SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
143 const void __user *, user_buffer, size_t, length)
144{
145 u8 local_buf[64];
146 void __iomem *io_addr;
147 void *buf;
148 unsigned long pfn;
149 long ret;
150
151 if (!zpci_is_enabled())
152 return -ENODEV;
153
154 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
155 return -EINVAL;
156
157 /*
158 * We only support write access to MIO capable devices if we are on
159 * a MIO enabled system. Otherwise we would have to check for every
160 * address if it is a special ZPCI_ADDR and would have to do
161 * a get_pfn() which we don't need for MIO capable devices. Currently
162 * ISM devices are the only devices without MIO support and there is no
163 * known need for accessing these from userspace.
164 */
165 if (static_branch_likely(&have_mio)) {
166 ret = __memcpy_toio_inuser((void __iomem *) mmio_addr,
167 user_buffer,
168 length);
169 return ret;
170 }
171
172 if (length > 64) {
173 buf = kmalloc(length, GFP_KERNEL);
174 if (!buf)
175 return -ENOMEM;
176 } else
177 buf = local_buf;
178
179 ret = get_pfn(mmio_addr, VM_WRITE, &pfn);
180 if (ret)
181 goto out;
182 io_addr = (void __iomem *)((pfn << PAGE_SHIFT) |
183 (mmio_addr & ~PAGE_MASK));
184
185 ret = -EFAULT;
186 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
187 goto out;
188
189 if (copy_from_user(buf, user_buffer, length))
190 goto out;
191
192 ret = zpci_memcpy_toio(io_addr, buf, length);
193out:
194 if (buf != local_buf)
195 kfree(buf);
196 return ret;
197}
198
199static inline int __pcilg_mio_inuser(
200 void __user *dst, const void __iomem *ioaddr,
201 u64 ulen, u8 *status)
202{
203 register u64 addr asm("2") = (u64 __force) ioaddr;
204 register u64 len asm("3") = ulen;
205 u64 cnt = ulen;
206 int shift = ulen * 8;
207 int cc = -ENXIO;
208 u64 val, tmp;
209
210 /*
211 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
212 * user space) into a register using pcilg then store these bytes at
213 * user address @dst
214 */
215 asm volatile (
216 " sacf 256\n"
217 "0: .insn rre,0xb9d60000,%[val],%[ioaddr]\n"
218 "1: ipm %[cc]\n"
219 " srl %[cc],28\n"
220 " ltr %[cc],%[cc]\n"
221 " jne 4f\n"
222 "2: ahi %[shift],-8\n"
223 " srlg %[tmp],%[val],0(%[shift])\n"
224 "3: stc %[tmp],0(%[dst])\n"
225 " aghi %[dst],1\n"
226 " brctg %[cnt],2b\n"
227 "4: sacf 768\n"
228 EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b)
229 :
230 [cc] "+d" (cc), [val] "=d" (val), [len] "+d" (len),
231 [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
232 [shift] "+d" (shift)
233 :
234 [ioaddr] "a" (addr)
235 : "cc", "memory");
236
237 /* did we write everything to the user space buffer? */
238 if (!cc && cnt != 0)
239 cc = -EFAULT;
240
241 *status = len >> 24 & 0xff;
242 return cc;
243}
244
245static inline int __memcpy_fromio_inuser(void __user *dst,
246 const void __iomem *src,
247 unsigned long n)
248{
249 int size, rc = 0;
250 u8 status;
251 mm_segment_t old_fs;
252
253 old_fs = enable_sacf_uaccess();
254 while (n > 0) {
255 size = zpci_get_max_write_size((u64 __force) src,
256 (u64 __force) dst, n,
257 ZPCI_MAX_READ_SIZE);
258 rc = __pcilg_mio_inuser(dst, src, size, &status);
259 if (rc)
260 break;
261 src += size;
262 dst += size;
263 n -= size;
264 }
265 disable_sacf_uaccess(old_fs);
266 if (rc)
267 zpci_err_mmio(rc, status, (__force u64) dst);
268 return rc;
269}
270
271SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
272 void __user *, user_buffer, size_t, length)
273{
274 u8 local_buf[64];
275 void __iomem *io_addr;
276 void *buf;
277 unsigned long pfn;
278 long ret;
279
280 if (!zpci_is_enabled())
281 return -ENODEV;
282
283 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
284 return -EINVAL;
285
286 /*
287 * We only support read access to MIO capable devices if we are on
288 * a MIO enabled system. Otherwise we would have to check for every
289 * address if it is a special ZPCI_ADDR and would have to do
290 * a get_pfn() which we don't need for MIO capable devices. Currently
291 * ISM devices are the only devices without MIO support and there is no
292 * known need for accessing these from userspace.
293 */
294 if (static_branch_likely(&have_mio)) {
295 ret = __memcpy_fromio_inuser(
296 user_buffer, (const void __iomem *)mmio_addr,
297 length);
298 return ret;
299 }
300
301 if (length > 64) {
302 buf = kmalloc(length, GFP_KERNEL);
303 if (!buf)
304 return -ENOMEM;
305 } else {
306 buf = local_buf;
307 }
308
309 ret = get_pfn(mmio_addr, VM_READ, &pfn);
310 if (ret)
311 goto out;
312 io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK));
313
314 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
315 ret = -EFAULT;
316 goto out;
317 }
318 ret = zpci_memcpy_fromio(buf, io_addr, length);
319 if (ret)
320 goto out;
321 if (copy_to_user(user_buffer, buf, length))
322 ret = -EFAULT;
323
324out:
325 if (buf != local_buf)
326 kfree(buf);
327 return ret;
328}