1#include <sys/types.h>
  2#include <sys/stat.h>
  3#include <sys/mman.h>
  4#include <unistd.h>
  5#include <errno.h>
  6#include <string.h>
  7
  8#include "liburing.h"
  9#include "barrier.h"
 10
 11static int __io_uring_get_cqe(struct io_uring *ring,
 12			      struct io_uring_cqe **cqe_ptr, int wait)
 13{
 14	struct io_uring_cq *cq = &ring->cq;
 15	const unsigned mask = *cq->kring_mask;
 16	unsigned head;
 17	int ret;
 18
 19	*cqe_ptr = NULL;
 20	head = *cq->khead;
 21	do {
 22		/*
 23		 * It's necessary to use a read_barrier() before reading
 24		 * the CQ tail, since the kernel updates it locklessly. The
 25		 * kernel has the matching store barrier for the update. The
 26		 * kernel also ensures that previous stores to CQEs are ordered
 27		 * with the tail update.
 28		 */
 29		read_barrier();
 30		if (head != *cq->ktail) {
 31			*cqe_ptr = &cq->cqes[head & mask];
 32			break;
 33		}
 34		if (!wait)
 35			break;
 36		ret = io_uring_enter(ring->ring_fd, 0, 1,
 37					IORING_ENTER_GETEVENTS, NULL);
 38		if (ret < 0)
 39			return -errno;
 40	} while (1);
 41
 42	return 0;
 43}
 44
 45/*
 46 * Return an IO completion, if one is readily available. Returns 0 with
 47 * cqe_ptr filled in on success, -errno on failure.
 48 */
 49int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
 50{
 51	return __io_uring_get_cqe(ring, cqe_ptr, 0);
 52}
 53
 54/*
 55 * Return an IO completion, waiting for it if necessary. Returns 0 with
 56 * cqe_ptr filled in on success, -errno on failure.
 57 */
 58int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
 59{
 60	return __io_uring_get_cqe(ring, cqe_ptr, 1);
 61}
 62
 63/*
 64 * Submit sqes acquired from io_uring_get_sqe() to the kernel.
 65 *
 66 * Returns number of sqes submitted
 67 */
 68int io_uring_submit(struct io_uring *ring)
 69{
 70	struct io_uring_sq *sq = &ring->sq;
 71	const unsigned mask = *sq->kring_mask;
 72	unsigned ktail, ktail_next, submitted, to_submit;
 73	int ret;
 74
 75	/*
 76	 * If we have pending IO in the kring, submit it first. We need a
 77	 * read barrier here to match the kernels store barrier when updating
 78	 * the SQ head.
 79	 */
 80	read_barrier();
 81	if (*sq->khead != *sq->ktail) {
 82		submitted = *sq->kring_entries;
 83		goto submit;
 84	}
 85
 86	if (sq->sqe_head == sq->sqe_tail)
 87		return 0;
 88
 89	/*
 90	 * Fill in sqes that we have queued up, adding them to the kernel ring
 91	 */
 92	submitted = 0;
 93	ktail = ktail_next = *sq->ktail;
 94	to_submit = sq->sqe_tail - sq->sqe_head;
 95	while (to_submit--) {
 96		ktail_next++;
 97		read_barrier();
 98
 99		sq->array[ktail & mask] = sq->sqe_head & mask;
100		ktail = ktail_next;
101
102		sq->sqe_head++;
103		submitted++;
104	}
105
106	if (!submitted)
107		return 0;
108
109	if (*sq->ktail != ktail) {
110		/*
111		 * First write barrier ensures that the SQE stores are updated
112		 * with the tail update. This is needed so that the kernel
113		 * will never see a tail update without the preceeding sQE
114		 * stores being done.
115		 */
116		write_barrier();
117		*sq->ktail = ktail;
118		/*
119		 * The kernel has the matching read barrier for reading the
120		 * SQ tail.
121		 */
122		write_barrier();
123	}
124
125submit:
126	ret = io_uring_enter(ring->ring_fd, submitted, 0,
127				IORING_ENTER_GETEVENTS, NULL);
128	if (ret < 0)
129		return -errno;
130
131	return ret;
132}
133
134/*
135 * Return an sqe to fill. Application must later call io_uring_submit()
136 * when it's ready to tell the kernel about it. The caller may call this
137 * function multiple times before calling io_uring_submit().
138 *
139 * Returns a vacant sqe, or NULL if we're full.
140 */
141struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
142{
143	struct io_uring_sq *sq = &ring->sq;
144	unsigned next = sq->sqe_tail + 1;
145	struct io_uring_sqe *sqe;
146
147	/*
148	 * All sqes are used
149	 */
150	if (next - sq->sqe_head > *sq->kring_entries)
151		return NULL;
152
153	sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask];
154	sq->sqe_tail = next;
155	return sqe;
156}