Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2014 IBM Corp.
4 */
5
6#include <linux/pci.h>
7#include <linux/slab.h>
8#include <linux/file.h>
9#include <misc/cxl.h>
10#include <linux/module.h>
11#include <linux/mount.h>
12#include <linux/pseudo_fs.h>
13#include <linux/sched/mm.h>
14#include <linux/mmu_context.h>
15
16#include "cxl.h"
17
18/*
19 * Since we want to track memory mappings to be able to force-unmap
20 * when the AFU is no longer reachable, we need an inode. For devices
21 * opened through the cxl user API, this is not a problem, but a
22 * userland process can also get a cxl fd through the cxl_get_fd()
23 * API, which is used by the cxlflash driver.
24 *
25 * Therefore we implement our own simple pseudo-filesystem and inode
26 * allocator. We don't use the anonymous inode, as we need the
27 * meta-data associated with it (address_space) and it is shared by
28 * other drivers/processes, so it could lead to cxl unmapping VMAs
29 * from random processes.
30 */
31
32#define CXL_PSEUDO_FS_MAGIC 0x1697697f
33
34static int cxl_fs_cnt;
35static struct vfsmount *cxl_vfs_mount;
36
37static int cxl_fs_init_fs_context(struct fs_context *fc)
38{
39 return init_pseudo(fc, CXL_PSEUDO_FS_MAGIC) ? 0 : -ENOMEM;
40}
41
42static struct file_system_type cxl_fs_type = {
43 .name = "cxl",
44 .owner = THIS_MODULE,
45 .init_fs_context = cxl_fs_init_fs_context,
46 .kill_sb = kill_anon_super,
47};
48
49
50void cxl_release_mapping(struct cxl_context *ctx)
51{
52 if (ctx->kernelapi && ctx->mapping)
53 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
54}
55
56static struct file *cxl_getfile(const char *name,
57 const struct file_operations *fops,
58 void *priv, int flags)
59{
60 struct file *file;
61 struct inode *inode;
62 int rc;
63
64 /* strongly inspired by anon_inode_getfile() */
65
66 if (fops->owner && !try_module_get(fops->owner))
67 return ERR_PTR(-ENOENT);
68
69 rc = simple_pin_fs(&cxl_fs_type, &cxl_vfs_mount, &cxl_fs_cnt);
70 if (rc < 0) {
71 pr_err("Cannot mount cxl pseudo filesystem: %d\n", rc);
72 file = ERR_PTR(rc);
73 goto err_module;
74 }
75
76 inode = alloc_anon_inode(cxl_vfs_mount->mnt_sb);
77 if (IS_ERR(inode)) {
78 file = ERR_CAST(inode);
79 goto err_fs;
80 }
81
82 file = alloc_file_pseudo(inode, cxl_vfs_mount, name,
83 flags & (O_ACCMODE | O_NONBLOCK), fops);
84 if (IS_ERR(file))
85 goto err_inode;
86
87 file->private_data = priv;
88
89 return file;
90
91err_inode:
92 iput(inode);
93err_fs:
94 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
95err_module:
96 module_put(fops->owner);
97 return file;
98}
99
100struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)
101{
102 struct cxl_afu *afu;
103 struct cxl_context *ctx;
104 int rc;
105
106 afu = cxl_pci_to_afu(dev);
107 if (IS_ERR(afu))
108 return ERR_CAST(afu);
109
110 ctx = cxl_context_alloc();
111 if (!ctx)
112 return ERR_PTR(-ENOMEM);
113
114 ctx->kernelapi = true;
115
116 /* Make it a slave context. We can promote it later? */
117 rc = cxl_context_init(ctx, afu, false);
118 if (rc)
119 goto err_ctx;
120
121 return ctx;
122
123err_ctx:
124 kfree(ctx);
125 return ERR_PTR(rc);
126}
127EXPORT_SYMBOL_GPL(cxl_dev_context_init);
128
129struct cxl_context *cxl_get_context(struct pci_dev *dev)
130{
131 return dev->dev.archdata.cxl_ctx;
132}
133EXPORT_SYMBOL_GPL(cxl_get_context);
134
135int cxl_release_context(struct cxl_context *ctx)
136{
137 if (ctx->status >= STARTED)
138 return -EBUSY;
139
140 cxl_context_free(ctx);
141
142 return 0;
143}
144EXPORT_SYMBOL_GPL(cxl_release_context);
145
146static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)
147{
148 __u16 range;
149 int r;
150
151 for (r = 0; r < CXL_IRQ_RANGES; r++) {
152 range = ctx->irqs.range[r];
153 if (num < range) {
154 return ctx->irqs.offset[r] + num;
155 }
156 num -= range;
157 }
158 return 0;
159}
160
161
162int cxl_set_priv(struct cxl_context *ctx, void *priv)
163{
164 if (!ctx)
165 return -EINVAL;
166
167 ctx->priv = priv;
168
169 return 0;
170}
171EXPORT_SYMBOL_GPL(cxl_set_priv);
172
173void *cxl_get_priv(struct cxl_context *ctx)
174{
175 if (!ctx)
176 return ERR_PTR(-EINVAL);
177
178 return ctx->priv;
179}
180EXPORT_SYMBOL_GPL(cxl_get_priv);
181
182int cxl_allocate_afu_irqs(struct cxl_context *ctx, int num)
183{
184 int res;
185 irq_hw_number_t hwirq;
186
187 if (num == 0)
188 num = ctx->afu->pp_irqs;
189 res = afu_allocate_irqs(ctx, num);
190 if (res)
191 return res;
192
193 if (!cpu_has_feature(CPU_FTR_HVMODE)) {
194 /* In a guest, the PSL interrupt is not multiplexed. It was
195 * allocated above, and we need to set its handler
196 */
197 hwirq = cxl_find_afu_irq(ctx, 0);
198 if (hwirq)
199 cxl_map_irq(ctx->afu->adapter, hwirq, cxl_ops->psl_interrupt, ctx, "psl");
200 }
201
202 if (ctx->status == STARTED) {
203 if (cxl_ops->update_ivtes)
204 cxl_ops->update_ivtes(ctx);
205 else WARN(1, "BUG: cxl_allocate_afu_irqs must be called prior to starting the context on this platform\n");
206 }
207
208 return res;
209}
210EXPORT_SYMBOL_GPL(cxl_allocate_afu_irqs);
211
212void cxl_free_afu_irqs(struct cxl_context *ctx)
213{
214 irq_hw_number_t hwirq;
215 unsigned int virq;
216
217 if (!cpu_has_feature(CPU_FTR_HVMODE)) {
218 hwirq = cxl_find_afu_irq(ctx, 0);
219 if (hwirq) {
220 virq = irq_find_mapping(NULL, hwirq);
221 if (virq)
222 cxl_unmap_irq(virq, ctx);
223 }
224 }
225 afu_irq_name_free(ctx);
226 cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
227}
228EXPORT_SYMBOL_GPL(cxl_free_afu_irqs);
229
230int cxl_map_afu_irq(struct cxl_context *ctx, int num,
231 irq_handler_t handler, void *cookie, char *name)
232{
233 irq_hw_number_t hwirq;
234
235 /*
236 * Find interrupt we are to register.
237 */
238 hwirq = cxl_find_afu_irq(ctx, num);
239 if (!hwirq)
240 return -ENOENT;
241
242 return cxl_map_irq(ctx->afu->adapter, hwirq, handler, cookie, name);
243}
244EXPORT_SYMBOL_GPL(cxl_map_afu_irq);
245
246void cxl_unmap_afu_irq(struct cxl_context *ctx, int num, void *cookie)
247{
248 irq_hw_number_t hwirq;
249 unsigned int virq;
250
251 hwirq = cxl_find_afu_irq(ctx, num);
252 if (!hwirq)
253 return;
254
255 virq = irq_find_mapping(NULL, hwirq);
256 if (virq)
257 cxl_unmap_irq(virq, cookie);
258}
259EXPORT_SYMBOL_GPL(cxl_unmap_afu_irq);
260
261/*
262 * Start a context
263 * Code here similar to afu_ioctl_start_work().
264 */
265int cxl_start_context(struct cxl_context *ctx, u64 wed,
266 struct task_struct *task)
267{
268 int rc = 0;
269 bool kernel = true;
270
271 pr_devel("%s: pe: %i\n", __func__, ctx->pe);
272
273 mutex_lock(&ctx->status_mutex);
274 if (ctx->status == STARTED)
275 goto out; /* already started */
276
277 /*
278 * Increment the mapped context count for adapter. This also checks
279 * if adapter_context_lock is taken.
280 */
281 rc = cxl_adapter_context_get(ctx->afu->adapter);
282 if (rc)
283 goto out;
284
285 if (task) {
286 ctx->pid = get_task_pid(task, PIDTYPE_PID);
287 kernel = false;
288
289 /* acquire a reference to the task's mm */
290 ctx->mm = get_task_mm(current);
291
292 /* ensure this mm_struct can't be freed */
293 cxl_context_mm_count_get(ctx);
294
295 if (ctx->mm) {
296 /* decrement the use count from above */
297 mmput(ctx->mm);
298 /* make TLBIs for this context global */
299 mm_context_add_copro(ctx->mm);
300 }
301 }
302
303 /*
304 * Increment driver use count. Enables global TLBIs for hash
305 * and callbacks to handle the segment table
306 */
307 cxl_ctx_get();
308
309 /* See the comment in afu_ioctl_start_work() */
310 smp_mb();
311
312 if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) {
313 put_pid(ctx->pid);
314 ctx->pid = NULL;
315 cxl_adapter_context_put(ctx->afu->adapter);
316 cxl_ctx_put();
317 if (task) {
318 cxl_context_mm_count_put(ctx);
319 if (ctx->mm)
320 mm_context_remove_copro(ctx->mm);
321 }
322 goto out;
323 }
324
325 ctx->status = STARTED;
326out:
327 mutex_unlock(&ctx->status_mutex);
328 return rc;
329}
330EXPORT_SYMBOL_GPL(cxl_start_context);
331
332int cxl_process_element(struct cxl_context *ctx)
333{
334 return ctx->external_pe;
335}
336EXPORT_SYMBOL_GPL(cxl_process_element);
337
338/* Stop a context. Returns 0 on success, otherwise -Errno */
339int cxl_stop_context(struct cxl_context *ctx)
340{
341 return __detach_context(ctx);
342}
343EXPORT_SYMBOL_GPL(cxl_stop_context);
344
345void cxl_set_master(struct cxl_context *ctx)
346{
347 ctx->master = true;
348}
349EXPORT_SYMBOL_GPL(cxl_set_master);
350
351/* wrappers around afu_* file ops which are EXPORTED */
352int cxl_fd_open(struct inode *inode, struct file *file)
353{
354 return afu_open(inode, file);
355}
356EXPORT_SYMBOL_GPL(cxl_fd_open);
357int cxl_fd_release(struct inode *inode, struct file *file)
358{
359 return afu_release(inode, file);
360}
361EXPORT_SYMBOL_GPL(cxl_fd_release);
362long cxl_fd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
363{
364 return afu_ioctl(file, cmd, arg);
365}
366EXPORT_SYMBOL_GPL(cxl_fd_ioctl);
367int cxl_fd_mmap(struct file *file, struct vm_area_struct *vm)
368{
369 return afu_mmap(file, vm);
370}
371EXPORT_SYMBOL_GPL(cxl_fd_mmap);
372__poll_t cxl_fd_poll(struct file *file, struct poll_table_struct *poll)
373{
374 return afu_poll(file, poll);
375}
376EXPORT_SYMBOL_GPL(cxl_fd_poll);
377ssize_t cxl_fd_read(struct file *file, char __user *buf, size_t count,
378 loff_t *off)
379{
380 return afu_read(file, buf, count, off);
381}
382EXPORT_SYMBOL_GPL(cxl_fd_read);
383
384#define PATCH_FOPS(NAME) if (!fops->NAME) fops->NAME = afu_fops.NAME
385
386/* Get a struct file and fd for a context and attach the ops */
387struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops,
388 int *fd)
389{
390 struct file *file;
391 int rc, flags, fdtmp;
392 char *name = NULL;
393
394 /* only allow one per context */
395 if (ctx->mapping)
396 return ERR_PTR(-EEXIST);
397
398 flags = O_RDWR | O_CLOEXEC;
399
400 /* This code is similar to anon_inode_getfd() */
401 rc = get_unused_fd_flags(flags);
402 if (rc < 0)
403 return ERR_PTR(rc);
404 fdtmp = rc;
405
406 /*
407 * Patch the file ops. Needs to be careful that this is rentrant safe.
408 */
409 if (fops) {
410 PATCH_FOPS(open);
411 PATCH_FOPS(poll);
412 PATCH_FOPS(read);
413 PATCH_FOPS(release);
414 PATCH_FOPS(unlocked_ioctl);
415 PATCH_FOPS(compat_ioctl);
416 PATCH_FOPS(mmap);
417 } else /* use default ops */
418 fops = (struct file_operations *)&afu_fops;
419
420 name = kasprintf(GFP_KERNEL, "cxl:%d", ctx->pe);
421 file = cxl_getfile(name, fops, ctx, flags);
422 kfree(name);
423 if (IS_ERR(file))
424 goto err_fd;
425
426 cxl_context_set_mapping(ctx, file->f_mapping);
427 *fd = fdtmp;
428 return file;
429
430err_fd:
431 put_unused_fd(fdtmp);
432 return NULL;
433}
434EXPORT_SYMBOL_GPL(cxl_get_fd);
435
436struct cxl_context *cxl_fops_get_context(struct file *file)
437{
438 return file->private_data;
439}
440EXPORT_SYMBOL_GPL(cxl_fops_get_context);
441
442void cxl_set_driver_ops(struct cxl_context *ctx,
443 struct cxl_afu_driver_ops *ops)
444{
445 WARN_ON(!ops->fetch_event || !ops->event_delivered);
446 atomic_set(&ctx->afu_driver_events, 0);
447 ctx->afu_driver_ops = ops;
448}
449EXPORT_SYMBOL_GPL(cxl_set_driver_ops);
450
451void cxl_context_events_pending(struct cxl_context *ctx,
452 unsigned int new_events)
453{
454 atomic_add(new_events, &ctx->afu_driver_events);
455 wake_up_all(&ctx->wq);
456}
457EXPORT_SYMBOL_GPL(cxl_context_events_pending);
458
459int cxl_start_work(struct cxl_context *ctx,
460 struct cxl_ioctl_start_work *work)
461{
462 int rc;
463
464 /* code taken from afu_ioctl_start_work */
465 if (!(work->flags & CXL_START_WORK_NUM_IRQS))
466 work->num_interrupts = ctx->afu->pp_irqs;
467 else if ((work->num_interrupts < ctx->afu->pp_irqs) ||
468 (work->num_interrupts > ctx->afu->irqs_max)) {
469 return -EINVAL;
470 }
471
472 rc = afu_register_irqs(ctx, work->num_interrupts);
473 if (rc)
474 return rc;
475
476 rc = cxl_start_context(ctx, work->work_element_descriptor, current);
477 if (rc < 0) {
478 afu_release_irqs(ctx, ctx);
479 return rc;
480 }
481
482 return 0;
483}
484EXPORT_SYMBOL_GPL(cxl_start_work);
485
486void __iomem *cxl_psa_map(struct cxl_context *ctx)
487{
488 if (ctx->status != STARTED)
489 return NULL;
490
491 pr_devel("%s: psn_phys%llx size:%llx\n",
492 __func__, ctx->psn_phys, ctx->psn_size);
493 return ioremap(ctx->psn_phys, ctx->psn_size);
494}
495EXPORT_SYMBOL_GPL(cxl_psa_map);
496
497void cxl_psa_unmap(void __iomem *addr)
498{
499 iounmap(addr);
500}
501EXPORT_SYMBOL_GPL(cxl_psa_unmap);
502
503int cxl_afu_reset(struct cxl_context *ctx)
504{
505 struct cxl_afu *afu = ctx->afu;
506 int rc;
507
508 rc = cxl_ops->afu_reset(afu);
509 if (rc)
510 return rc;
511
512 return cxl_ops->afu_check_and_enable(afu);
513}
514EXPORT_SYMBOL_GPL(cxl_afu_reset);
515
516void cxl_perst_reloads_same_image(struct cxl_afu *afu,
517 bool perst_reloads_same_image)
518{
519 afu->adapter->perst_same_image = perst_reloads_same_image;
520}
521EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image);
522
523ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)
524{
525 struct cxl_afu *afu = cxl_pci_to_afu(dev);
526 if (IS_ERR(afu))
527 return -ENODEV;
528
529 return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);
530}
531EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);