Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2014 IBM Corp.
4 */
5
6#include <linux/pci.h>
7#include <linux/slab.h>
8#include <linux/file.h>
9#include <misc/cxl.h>
10#include <linux/module.h>
11#include <linux/mount.h>
12#include <linux/pseudo_fs.h>
13#include <linux/sched/mm.h>
14#include <linux/mmu_context.h>
15#include <linux/irqdomain.h>
16
17#include "cxl.h"
18
19/*
20 * Since we want to track memory mappings to be able to force-unmap
21 * when the AFU is no longer reachable, we need an inode. For devices
22 * opened through the cxl user API, this is not a problem, but a
23 * userland process can also get a cxl fd through the cxl_get_fd()
24 * API, which is used by the cxlflash driver.
25 *
26 * Therefore we implement our own simple pseudo-filesystem and inode
27 * allocator. We don't use the anonymous inode, as we need the
28 * meta-data associated with it (address_space) and it is shared by
29 * other drivers/processes, so it could lead to cxl unmapping VMAs
30 * from random processes.
31 */
32
33#define CXL_PSEUDO_FS_MAGIC 0x1697697f
34
35static int cxl_fs_cnt;
36static struct vfsmount *cxl_vfs_mount;
37
38static int cxl_fs_init_fs_context(struct fs_context *fc)
39{
40 return init_pseudo(fc, CXL_PSEUDO_FS_MAGIC) ? 0 : -ENOMEM;
41}
42
43static struct file_system_type cxl_fs_type = {
44 .name = "cxl",
45 .owner = THIS_MODULE,
46 .init_fs_context = cxl_fs_init_fs_context,
47 .kill_sb = kill_anon_super,
48};
49
50
51void cxl_release_mapping(struct cxl_context *ctx)
52{
53 if (ctx->kernelapi && ctx->mapping)
54 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
55}
56
57static struct file *cxl_getfile(const char *name,
58 const struct file_operations *fops,
59 void *priv, int flags)
60{
61 struct file *file;
62 struct inode *inode;
63 int rc;
64
65 /* strongly inspired by anon_inode_getfile() */
66
67 if (fops->owner && !try_module_get(fops->owner))
68 return ERR_PTR(-ENOENT);
69
70 rc = simple_pin_fs(&cxl_fs_type, &cxl_vfs_mount, &cxl_fs_cnt);
71 if (rc < 0) {
72 pr_err("Cannot mount cxl pseudo filesystem: %d\n", rc);
73 file = ERR_PTR(rc);
74 goto err_module;
75 }
76
77 inode = alloc_anon_inode(cxl_vfs_mount->mnt_sb);
78 if (IS_ERR(inode)) {
79 file = ERR_CAST(inode);
80 goto err_fs;
81 }
82
83 file = alloc_file_pseudo(inode, cxl_vfs_mount, name,
84 flags & (O_ACCMODE | O_NONBLOCK), fops);
85 if (IS_ERR(file))
86 goto err_inode;
87
88 file->private_data = priv;
89
90 return file;
91
92err_inode:
93 iput(inode);
94err_fs:
95 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
96err_module:
97 module_put(fops->owner);
98 return file;
99}
100
101struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)
102{
103 struct cxl_afu *afu;
104 struct cxl_context *ctx;
105 int rc;
106
107 afu = cxl_pci_to_afu(dev);
108 if (IS_ERR(afu))
109 return ERR_CAST(afu);
110
111 ctx = cxl_context_alloc();
112 if (!ctx)
113 return ERR_PTR(-ENOMEM);
114
115 ctx->kernelapi = true;
116
117 /* Make it a slave context. We can promote it later? */
118 rc = cxl_context_init(ctx, afu, false);
119 if (rc)
120 goto err_ctx;
121
122 return ctx;
123
124err_ctx:
125 kfree(ctx);
126 return ERR_PTR(rc);
127}
128EXPORT_SYMBOL_GPL(cxl_dev_context_init);
129
130struct cxl_context *cxl_get_context(struct pci_dev *dev)
131{
132 return dev->dev.archdata.cxl_ctx;
133}
134EXPORT_SYMBOL_GPL(cxl_get_context);
135
136int cxl_release_context(struct cxl_context *ctx)
137{
138 if (ctx->status >= STARTED)
139 return -EBUSY;
140
141 cxl_context_free(ctx);
142
143 return 0;
144}
145EXPORT_SYMBOL_GPL(cxl_release_context);
146
147static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)
148{
149 __u16 range;
150 int r;
151
152 for (r = 0; r < CXL_IRQ_RANGES; r++) {
153 range = ctx->irqs.range[r];
154 if (num < range) {
155 return ctx->irqs.offset[r] + num;
156 }
157 num -= range;
158 }
159 return 0;
160}
161
162
163int cxl_set_priv(struct cxl_context *ctx, void *priv)
164{
165 if (!ctx)
166 return -EINVAL;
167
168 ctx->priv = priv;
169
170 return 0;
171}
172EXPORT_SYMBOL_GPL(cxl_set_priv);
173
174void *cxl_get_priv(struct cxl_context *ctx)
175{
176 if (!ctx)
177 return ERR_PTR(-EINVAL);
178
179 return ctx->priv;
180}
181EXPORT_SYMBOL_GPL(cxl_get_priv);
182
183int cxl_allocate_afu_irqs(struct cxl_context *ctx, int num)
184{
185 int res;
186 irq_hw_number_t hwirq;
187
188 if (num == 0)
189 num = ctx->afu->pp_irqs;
190 res = afu_allocate_irqs(ctx, num);
191 if (res)
192 return res;
193
194 if (!cpu_has_feature(CPU_FTR_HVMODE)) {
195 /* In a guest, the PSL interrupt is not multiplexed. It was
196 * allocated above, and we need to set its handler
197 */
198 hwirq = cxl_find_afu_irq(ctx, 0);
199 if (hwirq)
200 cxl_map_irq(ctx->afu->adapter, hwirq, cxl_ops->psl_interrupt, ctx, "psl");
201 }
202
203 if (ctx->status == STARTED) {
204 if (cxl_ops->update_ivtes)
205 cxl_ops->update_ivtes(ctx);
206 else WARN(1, "BUG: cxl_allocate_afu_irqs must be called prior to starting the context on this platform\n");
207 }
208
209 return res;
210}
211EXPORT_SYMBOL_GPL(cxl_allocate_afu_irqs);
212
213void cxl_free_afu_irqs(struct cxl_context *ctx)
214{
215 irq_hw_number_t hwirq;
216 unsigned int virq;
217
218 if (!cpu_has_feature(CPU_FTR_HVMODE)) {
219 hwirq = cxl_find_afu_irq(ctx, 0);
220 if (hwirq) {
221 virq = irq_find_mapping(NULL, hwirq);
222 if (virq)
223 cxl_unmap_irq(virq, ctx);
224 }
225 }
226 afu_irq_name_free(ctx);
227 cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
228}
229EXPORT_SYMBOL_GPL(cxl_free_afu_irqs);
230
231int cxl_map_afu_irq(struct cxl_context *ctx, int num,
232 irq_handler_t handler, void *cookie, char *name)
233{
234 irq_hw_number_t hwirq;
235
236 /*
237 * Find interrupt we are to register.
238 */
239 hwirq = cxl_find_afu_irq(ctx, num);
240 if (!hwirq)
241 return -ENOENT;
242
243 return cxl_map_irq(ctx->afu->adapter, hwirq, handler, cookie, name);
244}
245EXPORT_SYMBOL_GPL(cxl_map_afu_irq);
246
247void cxl_unmap_afu_irq(struct cxl_context *ctx, int num, void *cookie)
248{
249 irq_hw_number_t hwirq;
250 unsigned int virq;
251
252 hwirq = cxl_find_afu_irq(ctx, num);
253 if (!hwirq)
254 return;
255
256 virq = irq_find_mapping(NULL, hwirq);
257 if (virq)
258 cxl_unmap_irq(virq, cookie);
259}
260EXPORT_SYMBOL_GPL(cxl_unmap_afu_irq);
261
262/*
263 * Start a context
264 * Code here similar to afu_ioctl_start_work().
265 */
266int cxl_start_context(struct cxl_context *ctx, u64 wed,
267 struct task_struct *task)
268{
269 int rc = 0;
270 bool kernel = true;
271
272 pr_devel("%s: pe: %i\n", __func__, ctx->pe);
273
274 mutex_lock(&ctx->status_mutex);
275 if (ctx->status == STARTED)
276 goto out; /* already started */
277
278 /*
279 * Increment the mapped context count for adapter. This also checks
280 * if adapter_context_lock is taken.
281 */
282 rc = cxl_adapter_context_get(ctx->afu->adapter);
283 if (rc)
284 goto out;
285
286 if (task) {
287 ctx->pid = get_task_pid(task, PIDTYPE_PID);
288 kernel = false;
289
290 /* acquire a reference to the task's mm */
291 ctx->mm = get_task_mm(current);
292
293 /* ensure this mm_struct can't be freed */
294 cxl_context_mm_count_get(ctx);
295
296 if (ctx->mm) {
297 /* decrement the use count from above */
298 mmput(ctx->mm);
299 /* make TLBIs for this context global */
300 mm_context_add_copro(ctx->mm);
301 }
302 }
303
304 /*
305 * Increment driver use count. Enables global TLBIs for hash
306 * and callbacks to handle the segment table
307 */
308 cxl_ctx_get();
309
310 /* See the comment in afu_ioctl_start_work() */
311 smp_mb();
312
313 if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) {
314 put_pid(ctx->pid);
315 ctx->pid = NULL;
316 cxl_adapter_context_put(ctx->afu->adapter);
317 cxl_ctx_put();
318 if (task) {
319 cxl_context_mm_count_put(ctx);
320 if (ctx->mm)
321 mm_context_remove_copro(ctx->mm);
322 }
323 goto out;
324 }
325
326 ctx->status = STARTED;
327out:
328 mutex_unlock(&ctx->status_mutex);
329 return rc;
330}
331EXPORT_SYMBOL_GPL(cxl_start_context);
332
333int cxl_process_element(struct cxl_context *ctx)
334{
335 return ctx->external_pe;
336}
337EXPORT_SYMBOL_GPL(cxl_process_element);
338
339/* Stop a context. Returns 0 on success, otherwise -Errno */
340int cxl_stop_context(struct cxl_context *ctx)
341{
342 return __detach_context(ctx);
343}
344EXPORT_SYMBOL_GPL(cxl_stop_context);
345
346void cxl_set_master(struct cxl_context *ctx)
347{
348 ctx->master = true;
349}
350EXPORT_SYMBOL_GPL(cxl_set_master);
351
352/* wrappers around afu_* file ops which are EXPORTED */
353int cxl_fd_open(struct inode *inode, struct file *file)
354{
355 return afu_open(inode, file);
356}
357EXPORT_SYMBOL_GPL(cxl_fd_open);
358int cxl_fd_release(struct inode *inode, struct file *file)
359{
360 return afu_release(inode, file);
361}
362EXPORT_SYMBOL_GPL(cxl_fd_release);
363long cxl_fd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
364{
365 return afu_ioctl(file, cmd, arg);
366}
367EXPORT_SYMBOL_GPL(cxl_fd_ioctl);
368int cxl_fd_mmap(struct file *file, struct vm_area_struct *vm)
369{
370 return afu_mmap(file, vm);
371}
372EXPORT_SYMBOL_GPL(cxl_fd_mmap);
373__poll_t cxl_fd_poll(struct file *file, struct poll_table_struct *poll)
374{
375 return afu_poll(file, poll);
376}
377EXPORT_SYMBOL_GPL(cxl_fd_poll);
378ssize_t cxl_fd_read(struct file *file, char __user *buf, size_t count,
379 loff_t *off)
380{
381 return afu_read(file, buf, count, off);
382}
383EXPORT_SYMBOL_GPL(cxl_fd_read);
384
385#define PATCH_FOPS(NAME) if (!fops->NAME) fops->NAME = afu_fops.NAME
386
387/* Get a struct file and fd for a context and attach the ops */
388struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops,
389 int *fd)
390{
391 struct file *file;
392 int rc, flags, fdtmp;
393 char *name = NULL;
394
395 /* only allow one per context */
396 if (ctx->mapping)
397 return ERR_PTR(-EEXIST);
398
399 flags = O_RDWR | O_CLOEXEC;
400
401 /* This code is similar to anon_inode_getfd() */
402 rc = get_unused_fd_flags(flags);
403 if (rc < 0)
404 return ERR_PTR(rc);
405 fdtmp = rc;
406
407 /*
408 * Patch the file ops. Needs to be careful that this is rentrant safe.
409 */
410 if (fops) {
411 PATCH_FOPS(open);
412 PATCH_FOPS(poll);
413 PATCH_FOPS(read);
414 PATCH_FOPS(release);
415 PATCH_FOPS(unlocked_ioctl);
416 PATCH_FOPS(compat_ioctl);
417 PATCH_FOPS(mmap);
418 } else /* use default ops */
419 fops = (struct file_operations *)&afu_fops;
420
421 name = kasprintf(GFP_KERNEL, "cxl:%d", ctx->pe);
422 file = cxl_getfile(name, fops, ctx, flags);
423 kfree(name);
424 if (IS_ERR(file))
425 goto err_fd;
426
427 cxl_context_set_mapping(ctx, file->f_mapping);
428 *fd = fdtmp;
429 return file;
430
431err_fd:
432 put_unused_fd(fdtmp);
433 return NULL;
434}
435EXPORT_SYMBOL_GPL(cxl_get_fd);
436
437struct cxl_context *cxl_fops_get_context(struct file *file)
438{
439 return file->private_data;
440}
441EXPORT_SYMBOL_GPL(cxl_fops_get_context);
442
443void cxl_set_driver_ops(struct cxl_context *ctx,
444 struct cxl_afu_driver_ops *ops)
445{
446 WARN_ON(!ops->fetch_event || !ops->event_delivered);
447 atomic_set(&ctx->afu_driver_events, 0);
448 ctx->afu_driver_ops = ops;
449}
450EXPORT_SYMBOL_GPL(cxl_set_driver_ops);
451
452void cxl_context_events_pending(struct cxl_context *ctx,
453 unsigned int new_events)
454{
455 atomic_add(new_events, &ctx->afu_driver_events);
456 wake_up_all(&ctx->wq);
457}
458EXPORT_SYMBOL_GPL(cxl_context_events_pending);
459
460int cxl_start_work(struct cxl_context *ctx,
461 struct cxl_ioctl_start_work *work)
462{
463 int rc;
464
465 /* code taken from afu_ioctl_start_work */
466 if (!(work->flags & CXL_START_WORK_NUM_IRQS))
467 work->num_interrupts = ctx->afu->pp_irqs;
468 else if ((work->num_interrupts < ctx->afu->pp_irqs) ||
469 (work->num_interrupts > ctx->afu->irqs_max)) {
470 return -EINVAL;
471 }
472
473 rc = afu_register_irqs(ctx, work->num_interrupts);
474 if (rc)
475 return rc;
476
477 rc = cxl_start_context(ctx, work->work_element_descriptor, current);
478 if (rc < 0) {
479 afu_release_irqs(ctx, ctx);
480 return rc;
481 }
482
483 return 0;
484}
485EXPORT_SYMBOL_GPL(cxl_start_work);
486
487void __iomem *cxl_psa_map(struct cxl_context *ctx)
488{
489 if (ctx->status != STARTED)
490 return NULL;
491
492 pr_devel("%s: psn_phys%llx size:%llx\n",
493 __func__, ctx->psn_phys, ctx->psn_size);
494 return ioremap(ctx->psn_phys, ctx->psn_size);
495}
496EXPORT_SYMBOL_GPL(cxl_psa_map);
497
498void cxl_psa_unmap(void __iomem *addr)
499{
500 iounmap(addr);
501}
502EXPORT_SYMBOL_GPL(cxl_psa_unmap);
503
504int cxl_afu_reset(struct cxl_context *ctx)
505{
506 struct cxl_afu *afu = ctx->afu;
507 int rc;
508
509 rc = cxl_ops->afu_reset(afu);
510 if (rc)
511 return rc;
512
513 return cxl_ops->afu_check_and_enable(afu);
514}
515EXPORT_SYMBOL_GPL(cxl_afu_reset);
516
517void cxl_perst_reloads_same_image(struct cxl_afu *afu,
518 bool perst_reloads_same_image)
519{
520 afu->adapter->perst_same_image = perst_reloads_same_image;
521}
522EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image);
523
524ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)
525{
526 struct cxl_afu *afu = cxl_pci_to_afu(dev);
527 if (IS_ERR(afu))
528 return -ENODEV;
529
530 return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);
531}
532EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/pci.h>
11#include <linux/slab.h>
12#include <linux/file.h>
13#include <misc/cxl.h>
14#include <asm/pnv-pci.h>
15#include <linux/msi.h>
16#include <linux/module.h>
17#include <linux/mount.h>
18
19#include "cxl.h"
20
21/*
22 * Since we want to track memory mappings to be able to force-unmap
23 * when the AFU is no longer reachable, we need an inode. For devices
24 * opened through the cxl user API, this is not a problem, but a
25 * userland process can also get a cxl fd through the cxl_get_fd()
26 * API, which is used by the cxlflash driver.
27 *
28 * Therefore we implement our own simple pseudo-filesystem and inode
29 * allocator. We don't use the anonymous inode, as we need the
30 * meta-data associated with it (address_space) and it is shared by
31 * other drivers/processes, so it could lead to cxl unmapping VMAs
32 * from random processes.
33 */
34
35#define CXL_PSEUDO_FS_MAGIC 0x1697697f
36
37static int cxl_fs_cnt;
38static struct vfsmount *cxl_vfs_mount;
39
40static const struct dentry_operations cxl_fs_dops = {
41 .d_dname = simple_dname,
42};
43
44static struct dentry *cxl_fs_mount(struct file_system_type *fs_type, int flags,
45 const char *dev_name, void *data)
46{
47 return mount_pseudo(fs_type, "cxl:", NULL, &cxl_fs_dops,
48 CXL_PSEUDO_FS_MAGIC);
49}
50
51static struct file_system_type cxl_fs_type = {
52 .name = "cxl",
53 .owner = THIS_MODULE,
54 .mount = cxl_fs_mount,
55 .kill_sb = kill_anon_super,
56};
57
58
59void cxl_release_mapping(struct cxl_context *ctx)
60{
61 if (ctx->kernelapi && ctx->mapping)
62 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
63}
64
65static struct file *cxl_getfile(const char *name,
66 const struct file_operations *fops,
67 void *priv, int flags)
68{
69 struct qstr this;
70 struct path path;
71 struct file *file;
72 struct inode *inode = NULL;
73 int rc;
74
75 /* strongly inspired by anon_inode_getfile() */
76
77 if (fops->owner && !try_module_get(fops->owner))
78 return ERR_PTR(-ENOENT);
79
80 rc = simple_pin_fs(&cxl_fs_type, &cxl_vfs_mount, &cxl_fs_cnt);
81 if (rc < 0) {
82 pr_err("Cannot mount cxl pseudo filesystem: %d\n", rc);
83 file = ERR_PTR(rc);
84 goto err_module;
85 }
86
87 inode = alloc_anon_inode(cxl_vfs_mount->mnt_sb);
88 if (IS_ERR(inode)) {
89 file = ERR_CAST(inode);
90 goto err_fs;
91 }
92
93 file = ERR_PTR(-ENOMEM);
94 this.name = name;
95 this.len = strlen(name);
96 this.hash = 0;
97 path.dentry = d_alloc_pseudo(cxl_vfs_mount->mnt_sb, &this);
98 if (!path.dentry)
99 goto err_inode;
100
101 path.mnt = mntget(cxl_vfs_mount);
102 d_instantiate(path.dentry, inode);
103
104 file = alloc_file(&path, OPEN_FMODE(flags), fops);
105 if (IS_ERR(file))
106 goto err_dput;
107 file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
108 file->private_data = priv;
109
110 return file;
111
112err_dput:
113 path_put(&path);
114err_inode:
115 iput(inode);
116err_fs:
117 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
118err_module:
119 module_put(fops->owner);
120 return file;
121}
122
123struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)
124{
125 struct cxl_afu *afu;
126 struct cxl_context *ctx;
127 int rc;
128
129 afu = cxl_pci_to_afu(dev);
130 if (IS_ERR(afu))
131 return ERR_CAST(afu);
132
133 ctx = cxl_context_alloc();
134 if (!ctx)
135 return ERR_PTR(-ENOMEM);
136
137 ctx->kernelapi = true;
138
139 /* Make it a slave context. We can promote it later? */
140 rc = cxl_context_init(ctx, afu, false);
141 if (rc)
142 goto err_ctx;
143
144 return ctx;
145
146err_ctx:
147 kfree(ctx);
148 return ERR_PTR(rc);
149}
150EXPORT_SYMBOL_GPL(cxl_dev_context_init);
151
152struct cxl_context *cxl_get_context(struct pci_dev *dev)
153{
154 return dev->dev.archdata.cxl_ctx;
155}
156EXPORT_SYMBOL_GPL(cxl_get_context);
157
158int cxl_release_context(struct cxl_context *ctx)
159{
160 if (ctx->status >= STARTED)
161 return -EBUSY;
162
163 cxl_context_free(ctx);
164
165 return 0;
166}
167EXPORT_SYMBOL_GPL(cxl_release_context);
168
169static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)
170{
171 __u16 range;
172 int r;
173
174 for (r = 0; r < CXL_IRQ_RANGES; r++) {
175 range = ctx->irqs.range[r];
176 if (num < range) {
177 return ctx->irqs.offset[r] + num;
178 }
179 num -= range;
180 }
181 return 0;
182}
183
184int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq)
185{
186 if (*ctx == NULL || *afu_irq == 0) {
187 *afu_irq = 1;
188 *ctx = cxl_get_context(pdev);
189 } else {
190 (*afu_irq)++;
191 if (*afu_irq > cxl_get_max_irqs_per_process(pdev)) {
192 *ctx = list_next_entry(*ctx, extra_irq_contexts);
193 *afu_irq = 1;
194 }
195 }
196 return cxl_find_afu_irq(*ctx, *afu_irq);
197}
198/* Exported via cxl_base */
199
200int cxl_set_priv(struct cxl_context *ctx, void *priv)
201{
202 if (!ctx)
203 return -EINVAL;
204
205 ctx->priv = priv;
206
207 return 0;
208}
209EXPORT_SYMBOL_GPL(cxl_set_priv);
210
211void *cxl_get_priv(struct cxl_context *ctx)
212{
213 if (!ctx)
214 return ERR_PTR(-EINVAL);
215
216 return ctx->priv;
217}
218EXPORT_SYMBOL_GPL(cxl_get_priv);
219
220int cxl_allocate_afu_irqs(struct cxl_context *ctx, int num)
221{
222 int res;
223 irq_hw_number_t hwirq;
224
225 if (num == 0)
226 num = ctx->afu->pp_irqs;
227 res = afu_allocate_irqs(ctx, num);
228 if (res)
229 return res;
230
231 if (!cpu_has_feature(CPU_FTR_HVMODE)) {
232 /* In a guest, the PSL interrupt is not multiplexed. It was
233 * allocated above, and we need to set its handler
234 */
235 hwirq = cxl_find_afu_irq(ctx, 0);
236 if (hwirq)
237 cxl_map_irq(ctx->afu->adapter, hwirq, cxl_ops->psl_interrupt, ctx, "psl");
238 }
239
240 if (ctx->status == STARTED) {
241 if (cxl_ops->update_ivtes)
242 cxl_ops->update_ivtes(ctx);
243 else WARN(1, "BUG: cxl_allocate_afu_irqs must be called prior to starting the context on this platform\n");
244 }
245
246 return res;
247}
248EXPORT_SYMBOL_GPL(cxl_allocate_afu_irqs);
249
250void cxl_free_afu_irqs(struct cxl_context *ctx)
251{
252 irq_hw_number_t hwirq;
253 unsigned int virq;
254
255 if (!cpu_has_feature(CPU_FTR_HVMODE)) {
256 hwirq = cxl_find_afu_irq(ctx, 0);
257 if (hwirq) {
258 virq = irq_find_mapping(NULL, hwirq);
259 if (virq)
260 cxl_unmap_irq(virq, ctx);
261 }
262 }
263 afu_irq_name_free(ctx);
264 cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
265}
266EXPORT_SYMBOL_GPL(cxl_free_afu_irqs);
267
268int cxl_map_afu_irq(struct cxl_context *ctx, int num,
269 irq_handler_t handler, void *cookie, char *name)
270{
271 irq_hw_number_t hwirq;
272
273 /*
274 * Find interrupt we are to register.
275 */
276 hwirq = cxl_find_afu_irq(ctx, num);
277 if (!hwirq)
278 return -ENOENT;
279
280 return cxl_map_irq(ctx->afu->adapter, hwirq, handler, cookie, name);
281}
282EXPORT_SYMBOL_GPL(cxl_map_afu_irq);
283
284void cxl_unmap_afu_irq(struct cxl_context *ctx, int num, void *cookie)
285{
286 irq_hw_number_t hwirq;
287 unsigned int virq;
288
289 hwirq = cxl_find_afu_irq(ctx, num);
290 if (!hwirq)
291 return;
292
293 virq = irq_find_mapping(NULL, hwirq);
294 if (virq)
295 cxl_unmap_irq(virq, cookie);
296}
297EXPORT_SYMBOL_GPL(cxl_unmap_afu_irq);
298
299/*
300 * Start a context
301 * Code here similar to afu_ioctl_start_work().
302 */
303int cxl_start_context(struct cxl_context *ctx, u64 wed,
304 struct task_struct *task)
305{
306 int rc = 0;
307 bool kernel = true;
308
309 pr_devel("%s: pe: %i\n", __func__, ctx->pe);
310
311 mutex_lock(&ctx->status_mutex);
312 if (ctx->status == STARTED)
313 goto out; /* already started */
314
315 /*
316 * Increment the mapped context count for adapter. This also checks
317 * if adapter_context_lock is taken.
318 */
319 rc = cxl_adapter_context_get(ctx->afu->adapter);
320 if (rc)
321 goto out;
322
323 if (task) {
324 ctx->pid = get_task_pid(task, PIDTYPE_PID);
325 ctx->glpid = get_task_pid(task->group_leader, PIDTYPE_PID);
326 kernel = false;
327 ctx->real_mode = false;
328 }
329
330 cxl_ctx_get();
331
332 if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) {
333 put_pid(ctx->glpid);
334 put_pid(ctx->pid);
335 ctx->glpid = ctx->pid = NULL;
336 cxl_adapter_context_put(ctx->afu->adapter);
337 cxl_ctx_put();
338 goto out;
339 }
340
341 ctx->status = STARTED;
342out:
343 mutex_unlock(&ctx->status_mutex);
344 return rc;
345}
346EXPORT_SYMBOL_GPL(cxl_start_context);
347
348int cxl_process_element(struct cxl_context *ctx)
349{
350 return ctx->external_pe;
351}
352EXPORT_SYMBOL_GPL(cxl_process_element);
353
354/* Stop a context. Returns 0 on success, otherwise -Errno */
355int cxl_stop_context(struct cxl_context *ctx)
356{
357 return __detach_context(ctx);
358}
359EXPORT_SYMBOL_GPL(cxl_stop_context);
360
361void cxl_set_master(struct cxl_context *ctx)
362{
363 ctx->master = true;
364}
365EXPORT_SYMBOL_GPL(cxl_set_master);
366
367int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode)
368{
369 if (ctx->status == STARTED) {
370 /*
371 * We could potentially update the PE and issue an update LLCMD
372 * to support this, but it doesn't seem to have a good use case
373 * since it's trivial to just create a second kernel context
374 * with different translation modes, so until someone convinces
375 * me otherwise:
376 */
377 return -EBUSY;
378 }
379
380 ctx->real_mode = real_mode;
381 return 0;
382}
383EXPORT_SYMBOL_GPL(cxl_set_translation_mode);
384
385/* wrappers around afu_* file ops which are EXPORTED */
386int cxl_fd_open(struct inode *inode, struct file *file)
387{
388 return afu_open(inode, file);
389}
390EXPORT_SYMBOL_GPL(cxl_fd_open);
391int cxl_fd_release(struct inode *inode, struct file *file)
392{
393 return afu_release(inode, file);
394}
395EXPORT_SYMBOL_GPL(cxl_fd_release);
396long cxl_fd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
397{
398 return afu_ioctl(file, cmd, arg);
399}
400EXPORT_SYMBOL_GPL(cxl_fd_ioctl);
401int cxl_fd_mmap(struct file *file, struct vm_area_struct *vm)
402{
403 return afu_mmap(file, vm);
404}
405EXPORT_SYMBOL_GPL(cxl_fd_mmap);
406unsigned int cxl_fd_poll(struct file *file, struct poll_table_struct *poll)
407{
408 return afu_poll(file, poll);
409}
410EXPORT_SYMBOL_GPL(cxl_fd_poll);
411ssize_t cxl_fd_read(struct file *file, char __user *buf, size_t count,
412 loff_t *off)
413{
414 return afu_read(file, buf, count, off);
415}
416EXPORT_SYMBOL_GPL(cxl_fd_read);
417
418#define PATCH_FOPS(NAME) if (!fops->NAME) fops->NAME = afu_fops.NAME
419
420/* Get a struct file and fd for a context and attach the ops */
421struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops,
422 int *fd)
423{
424 struct file *file;
425 int rc, flags, fdtmp;
426 char *name = NULL;
427
428 /* only allow one per context */
429 if (ctx->mapping)
430 return ERR_PTR(-EEXIST);
431
432 flags = O_RDWR | O_CLOEXEC;
433
434 /* This code is similar to anon_inode_getfd() */
435 rc = get_unused_fd_flags(flags);
436 if (rc < 0)
437 return ERR_PTR(rc);
438 fdtmp = rc;
439
440 /*
441 * Patch the file ops. Needs to be careful that this is rentrant safe.
442 */
443 if (fops) {
444 PATCH_FOPS(open);
445 PATCH_FOPS(poll);
446 PATCH_FOPS(read);
447 PATCH_FOPS(release);
448 PATCH_FOPS(unlocked_ioctl);
449 PATCH_FOPS(compat_ioctl);
450 PATCH_FOPS(mmap);
451 } else /* use default ops */
452 fops = (struct file_operations *)&afu_fops;
453
454 name = kasprintf(GFP_KERNEL, "cxl:%d", ctx->pe);
455 file = cxl_getfile(name, fops, ctx, flags);
456 kfree(name);
457 if (IS_ERR(file))
458 goto err_fd;
459
460 cxl_context_set_mapping(ctx, file->f_mapping);
461 *fd = fdtmp;
462 return file;
463
464err_fd:
465 put_unused_fd(fdtmp);
466 return NULL;
467}
468EXPORT_SYMBOL_GPL(cxl_get_fd);
469
470struct cxl_context *cxl_fops_get_context(struct file *file)
471{
472 return file->private_data;
473}
474EXPORT_SYMBOL_GPL(cxl_fops_get_context);
475
476void cxl_set_driver_ops(struct cxl_context *ctx,
477 struct cxl_afu_driver_ops *ops)
478{
479 WARN_ON(!ops->fetch_event || !ops->event_delivered);
480 atomic_set(&ctx->afu_driver_events, 0);
481 ctx->afu_driver_ops = ops;
482}
483EXPORT_SYMBOL_GPL(cxl_set_driver_ops);
484
485void cxl_context_events_pending(struct cxl_context *ctx,
486 unsigned int new_events)
487{
488 atomic_add(new_events, &ctx->afu_driver_events);
489 wake_up_all(&ctx->wq);
490}
491EXPORT_SYMBOL_GPL(cxl_context_events_pending);
492
493int cxl_start_work(struct cxl_context *ctx,
494 struct cxl_ioctl_start_work *work)
495{
496 int rc;
497
498 /* code taken from afu_ioctl_start_work */
499 if (!(work->flags & CXL_START_WORK_NUM_IRQS))
500 work->num_interrupts = ctx->afu->pp_irqs;
501 else if ((work->num_interrupts < ctx->afu->pp_irqs) ||
502 (work->num_interrupts > ctx->afu->irqs_max)) {
503 return -EINVAL;
504 }
505
506 rc = afu_register_irqs(ctx, work->num_interrupts);
507 if (rc)
508 return rc;
509
510 rc = cxl_start_context(ctx, work->work_element_descriptor, current);
511 if (rc < 0) {
512 afu_release_irqs(ctx, ctx);
513 return rc;
514 }
515
516 return 0;
517}
518EXPORT_SYMBOL_GPL(cxl_start_work);
519
520void __iomem *cxl_psa_map(struct cxl_context *ctx)
521{
522 if (ctx->status != STARTED)
523 return NULL;
524
525 pr_devel("%s: psn_phys%llx size:%llx\n",
526 __func__, ctx->psn_phys, ctx->psn_size);
527 return ioremap(ctx->psn_phys, ctx->psn_size);
528}
529EXPORT_SYMBOL_GPL(cxl_psa_map);
530
531void cxl_psa_unmap(void __iomem *addr)
532{
533 iounmap(addr);
534}
535EXPORT_SYMBOL_GPL(cxl_psa_unmap);
536
537int cxl_afu_reset(struct cxl_context *ctx)
538{
539 struct cxl_afu *afu = ctx->afu;
540 int rc;
541
542 rc = cxl_ops->afu_reset(afu);
543 if (rc)
544 return rc;
545
546 return cxl_ops->afu_check_and_enable(afu);
547}
548EXPORT_SYMBOL_GPL(cxl_afu_reset);
549
550void cxl_perst_reloads_same_image(struct cxl_afu *afu,
551 bool perst_reloads_same_image)
552{
553 afu->adapter->perst_same_image = perst_reloads_same_image;
554}
555EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image);
556
557ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)
558{
559 struct cxl_afu *afu = cxl_pci_to_afu(dev);
560 if (IS_ERR(afu))
561 return -ENODEV;
562
563 return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);
564}
565EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);
566
567int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs)
568{
569 struct cxl_afu *afu = cxl_pci_to_afu(dev);
570 if (IS_ERR(afu))
571 return -ENODEV;
572
573 if (irqs > afu->adapter->user_irqs)
574 return -EINVAL;
575
576 /* Limit user_irqs to prevent the user increasing this via sysfs */
577 afu->adapter->user_irqs = irqs;
578 afu->irqs_max = irqs;
579
580 return 0;
581}
582EXPORT_SYMBOL_GPL(cxl_set_max_irqs_per_process);
583
584int cxl_get_max_irqs_per_process(struct pci_dev *dev)
585{
586 struct cxl_afu *afu = cxl_pci_to_afu(dev);
587 if (IS_ERR(afu))
588 return -ENODEV;
589
590 return afu->irqs_max;
591}
592EXPORT_SYMBOL_GPL(cxl_get_max_irqs_per_process);
593
594/*
595 * This is a special interrupt allocation routine called from the PHB's MSI
596 * setup function. When capi interrupts are allocated in this manner they must
597 * still be associated with a running context, but since the MSI APIs have no
598 * way to specify this we use the default context associated with the device.
599 *
600 * The Mellanox CX4 has a hardware limitation that restricts the maximum AFU
601 * interrupt number, so in order to overcome this their driver informs us of
602 * the restriction by setting the maximum interrupts per context, and we
603 * allocate additional contexts as necessary so that we can keep the AFU
604 * interrupt number within the supported range.
605 */
606int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
607{
608 struct cxl_context *ctx, *new_ctx, *default_ctx;
609 int remaining;
610 int rc;
611
612 ctx = default_ctx = cxl_get_context(pdev);
613 if (WARN_ON(!default_ctx))
614 return -ENODEV;
615
616 remaining = nvec;
617 while (remaining > 0) {
618 rc = cxl_allocate_afu_irqs(ctx, min(remaining, ctx->afu->irqs_max));
619 if (rc) {
620 pr_warn("%s: Failed to find enough free MSIs\n", pci_name(pdev));
621 return rc;
622 }
623 remaining -= ctx->afu->irqs_max;
624
625 if (ctx != default_ctx && default_ctx->status == STARTED) {
626 WARN_ON(cxl_start_context(ctx,
627 be64_to_cpu(default_ctx->elem->common.wed),
628 NULL));
629 }
630
631 if (remaining > 0) {
632 new_ctx = cxl_dev_context_init(pdev);
633 if (IS_ERR(new_ctx)) {
634 pr_warn("%s: Failed to allocate enough contexts for MSIs\n", pci_name(pdev));
635 return -ENOSPC;
636 }
637 list_add(&new_ctx->extra_irq_contexts, &ctx->extra_irq_contexts);
638 ctx = new_ctx;
639 }
640 }
641
642 return 0;
643}
644/* Exported via cxl_base */
645
646void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev)
647{
648 struct cxl_context *ctx, *pos, *tmp;
649
650 ctx = cxl_get_context(pdev);
651 if (WARN_ON(!ctx))
652 return;
653
654 cxl_free_afu_irqs(ctx);
655 list_for_each_entry_safe(pos, tmp, &ctx->extra_irq_contexts, extra_irq_contexts) {
656 cxl_stop_context(pos);
657 cxl_free_afu_irqs(pos);
658 list_del(&pos->extra_irq_contexts);
659 cxl_release_context(pos);
660 }
661}
662/* Exported via cxl_base */