1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * VFIO generic eventfd code for IRQFD support.
  4 * Derived from drivers/vfio/pci/vfio_pci_intrs.c
  5 *
  6 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
  7 *     Author: Alex Williamson <alex.williamson@redhat.com>
  8 */
  9
 10#include <linux/vfio.h>
 11#include <linux/eventfd.h>
 12#include <linux/file.h>
 13#include <linux/module.h>
 14#include <linux/slab.h>
 15#include "vfio.h"
 16
 17static struct workqueue_struct *vfio_irqfd_cleanup_wq;
 18static DEFINE_SPINLOCK(virqfd_lock);
 19
 20int __init vfio_virqfd_init(void)
 21{
 22	vfio_irqfd_cleanup_wq =
 23		create_singlethread_workqueue("vfio-irqfd-cleanup");
 24	if (!vfio_irqfd_cleanup_wq)
 25		return -ENOMEM;
 26
 27	return 0;
 28}
 29
 30void vfio_virqfd_exit(void)
 31{
 32	destroy_workqueue(vfio_irqfd_cleanup_wq);
 33}
 34
 35static void virqfd_deactivate(struct virqfd *virqfd)
 36{
 37	queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown);
 38}
 39
 40static int virqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
 41{
 42	struct virqfd *virqfd = container_of(wait, struct virqfd, wait);
 43	__poll_t flags = key_to_poll(key);
 44
 45	if (flags & EPOLLIN) {
 46		u64 cnt;
 47		eventfd_ctx_do_read(virqfd->eventfd, &cnt);
 48
 49		/* An event has been signaled, call function */
 50		if ((!virqfd->handler ||
 51		     virqfd->handler(virqfd->opaque, virqfd->data)) &&
 52		    virqfd->thread)
 53			schedule_work(&virqfd->inject);
 54	}
 55
 56	if (flags & EPOLLHUP) {
 57		unsigned long flags;
 58		spin_lock_irqsave(&virqfd_lock, flags);
 59
 60		/*
 61		 * The eventfd is closing, if the virqfd has not yet been
 62		 * queued for release, as determined by testing whether the
 63		 * virqfd pointer to it is still valid, queue it now.  As
 64		 * with kvm irqfds, we know we won't race against the virqfd
 65		 * going away because we hold the lock to get here.
 66		 */
 67		if (*(virqfd->pvirqfd) == virqfd) {
 68			*(virqfd->pvirqfd) = NULL;
 69			virqfd_deactivate(virqfd);
 70		}
 71
 72		spin_unlock_irqrestore(&virqfd_lock, flags);
 73	}
 74
 75	return 0;
 76}
 77
 78static void virqfd_ptable_queue_proc(struct file *file,
 79				     wait_queue_head_t *wqh, poll_table *pt)
 80{
 81	struct virqfd *virqfd = container_of(pt, struct virqfd, pt);
 82	add_wait_queue(wqh, &virqfd->wait);
 83}
 84
 85static void virqfd_shutdown(struct work_struct *work)
 86{
 87	struct virqfd *virqfd = container_of(work, struct virqfd, shutdown);
 88	u64 cnt;
 89
 90	eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt);
 91	flush_work(&virqfd->inject);
 92	eventfd_ctx_put(virqfd->eventfd);
 93
 94	kfree(virqfd);
 95}
 96
 97static void virqfd_inject(struct work_struct *work)
 98{
 99	struct virqfd *virqfd = container_of(work, struct virqfd, inject);
100	if (virqfd->thread)
101		virqfd->thread(virqfd->opaque, virqfd->data);
102}
103
104static void virqfd_flush_inject(struct work_struct *work)
105{
106	struct virqfd *virqfd = container_of(work, struct virqfd, flush_inject);
107
108	flush_work(&virqfd->inject);
109}
110
111int vfio_virqfd_enable(void *opaque,
112		       int (*handler)(void *, void *),
113		       void (*thread)(void *, void *),
114		       void *data, struct virqfd **pvirqfd, int fd)
115{
116	struct eventfd_ctx *ctx;
117	struct virqfd *virqfd;
118	int ret = 0;
119	__poll_t events;
120
121	virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL_ACCOUNT);
122	if (!virqfd)
123		return -ENOMEM;
124
125	virqfd->pvirqfd = pvirqfd;
126	virqfd->opaque = opaque;
127	virqfd->handler = handler;
128	virqfd->thread = thread;
129	virqfd->data = data;
130
131	INIT_WORK(&virqfd->shutdown, virqfd_shutdown);
132	INIT_WORK(&virqfd->inject, virqfd_inject);
133	INIT_WORK(&virqfd->flush_inject, virqfd_flush_inject);
134
135	CLASS(fd, irqfd)(fd);
136	if (fd_empty(irqfd)) {
137		ret = -EBADF;
138		goto err_fd;
139	}
140
141	ctx = eventfd_ctx_fileget(fd_file(irqfd));
142	if (IS_ERR(ctx)) {
143		ret = PTR_ERR(ctx);
144		goto err_fd;
145	}
146
147	virqfd->eventfd = ctx;
148
149	/*
150	 * virqfds can be released by closing the eventfd or directly
151	 * through ioctl.  These are both done through a workqueue, so
152	 * we update the pointer to the virqfd under lock to avoid
153	 * pushing multiple jobs to release the same virqfd.
154	 */
155	spin_lock_irq(&virqfd_lock);
156
157	if (*pvirqfd) {
158		spin_unlock_irq(&virqfd_lock);
159		ret = -EBUSY;
160		goto err_busy;
161	}
162	*pvirqfd = virqfd;
163
164	spin_unlock_irq(&virqfd_lock);
165
166	/*
167	 * Install our own custom wake-up handling so we are notified via
168	 * a callback whenever someone signals the underlying eventfd.
169	 */
170	init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup);
171	init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc);
172
173	events = vfs_poll(fd_file(irqfd), &virqfd->pt);
174
175	/*
176	 * Check if there was an event already pending on the eventfd
177	 * before we registered and trigger it as if we didn't miss it.
178	 */
179	if (events & EPOLLIN) {
180		if ((!handler || handler(opaque, data)) && thread)
181			schedule_work(&virqfd->inject);
182	}
183	return 0;
184err_busy:
185	eventfd_ctx_put(ctx);
186err_fd:
187	kfree(virqfd);
188
189	return ret;
190}
191EXPORT_SYMBOL_GPL(vfio_virqfd_enable);
192
193void vfio_virqfd_disable(struct virqfd **pvirqfd)
194{
195	unsigned long flags;
196
197	spin_lock_irqsave(&virqfd_lock, flags);
198
199	if (*pvirqfd) {
200		virqfd_deactivate(*pvirqfd);
201		*pvirqfd = NULL;
202	}
203
204	spin_unlock_irqrestore(&virqfd_lock, flags);
205
206	/*
207	 * Block until we know all outstanding shutdown jobs have completed.
208	 * Even if we don't queue the job, flush the wq to be sure it's
209	 * been released.
210	 */
211	flush_workqueue(vfio_irqfd_cleanup_wq);
212}
213EXPORT_SYMBOL_GPL(vfio_virqfd_disable);
214
215void vfio_virqfd_flush_thread(struct virqfd **pvirqfd)
216{
217	unsigned long flags;
218
219	spin_lock_irqsave(&virqfd_lock, flags);
220	if (*pvirqfd && (*pvirqfd)->thread)
221		queue_work(vfio_irqfd_cleanup_wq, &(*pvirqfd)->flush_inject);
222	spin_unlock_irqrestore(&virqfd_lock, flags);
223
224	flush_workqueue(vfio_irqfd_cleanup_wq);
225}
226EXPORT_SYMBOL_GPL(vfio_virqfd_flush_thread);