1// SPDX-License-Identifier: GPL-2.0-or-later
  2/* Iterator helpers.
  3 *
  4 * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved.
  5 * Written by David Howells (dhowells@redhat.com)
  6 */
  7
  8#include <linux/export.h>
  9#include <linux/slab.h>
 10#include <linux/mm.h>
 11#include <linux/uio.h>
 12#include <linux/scatterlist.h>
 13#include <linux/netfs.h>
 14#include "internal.h"
 15
 16/**
 17 * netfs_extract_user_iter - Extract the pages from a user iterator into a bvec
 18 * @orig: The original iterator
 19 * @orig_len: The amount of iterator to copy
 20 * @new: The iterator to be set up
 21 * @extraction_flags: Flags to qualify the request
 22 *
 23 * Extract the page fragments from the given amount of the source iterator and
 24 * build up a second iterator that refers to all of those bits.  This allows
 25 * the original iterator to disposed of.
 26 *
 27 * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be
 28 * allowed on the pages extracted.
 29 *
 30 * On success, the number of elements in the bvec is returned, the original
 31 * iterator will have been advanced by the amount extracted.
 32 *
 33 * The iov_iter_extract_mode() function should be used to query how cleanup
 34 * should be performed.
 35 */
 36ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
 37				struct iov_iter *new,
 38				iov_iter_extraction_t extraction_flags)
 39{
 40	struct bio_vec *bv = NULL;
 41	struct page **pages;
 42	unsigned int cur_npages;
 43	unsigned int max_pages;
 44	unsigned int npages = 0;
 45	unsigned int i;
 46	ssize_t ret;
 47	size_t count = orig_len, offset, len;
 48	size_t bv_size, pg_size;
 49
 50	if (WARN_ON_ONCE(!iter_is_ubuf(orig) && !iter_is_iovec(orig)))
 51		return -EIO;
 52
 53	max_pages = iov_iter_npages(orig, INT_MAX);
 54	bv_size = array_size(max_pages, sizeof(*bv));
 55	bv = kvmalloc(bv_size, GFP_KERNEL);
 56	if (!bv)
 57		return -ENOMEM;
 58
 59	/* Put the page list at the end of the bvec list storage.  bvec
 60	 * elements are larger than page pointers, so as long as we work
 61	 * 0->last, we should be fine.
 62	 */
 63	pg_size = array_size(max_pages, sizeof(*pages));
 64	pages = (void *)bv + bv_size - pg_size;
 65
 66	while (count && npages < max_pages) {
 67		ret = iov_iter_extract_pages(orig, &pages, count,
 68					     max_pages - npages, extraction_flags,
 69					     &offset);
 70		if (ret < 0) {
 71			pr_err("Couldn't get user pages (rc=%zd)\n", ret);
 72			break;
 73		}
 74
 75		if (ret > count) {
 76			pr_err("get_pages rc=%zd more than %zu\n", ret, count);
 77			break;
 78		}
 79
 80		count -= ret;
 81		ret += offset;
 82		cur_npages = DIV_ROUND_UP(ret, PAGE_SIZE);
 83
 84		if (npages + cur_npages > max_pages) {
 85			pr_err("Out of bvec array capacity (%u vs %u)\n",
 86			       npages + cur_npages, max_pages);
 87			break;
 88		}
 89
 90		for (i = 0; i < cur_npages; i++) {
 91			len = ret > PAGE_SIZE ? PAGE_SIZE : ret;
 92			bvec_set_page(bv + npages + i, *pages++, len - offset, offset);
 93			ret -= len;
 94			offset = 0;
 95		}
 96
 97		npages += cur_npages;
 98	}
 99
100	iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count);
101	return npages;
102}
103EXPORT_SYMBOL_GPL(netfs_extract_user_iter);
104
105/*
106 * Select the span of a bvec iterator we're going to use.  Limit it by both maximum
107 * size and maximum number of segments.  Returns the size of the span in bytes.
108 */
109static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset,
110			       size_t max_size, size_t max_segs)
111{
112	const struct bio_vec *bvecs = iter->bvec;
113	unsigned int nbv = iter->nr_segs, ix = 0, nsegs = 0;
114	size_t len, span = 0, n = iter->count;
115	size_t skip = iter->iov_offset + start_offset;
116
117	if (WARN_ON(!iov_iter_is_bvec(iter)) ||
118	    WARN_ON(start_offset > n) ||
119	    n == 0)
120		return 0;
121
122	while (n && ix < nbv && skip) {
123		len = bvecs[ix].bv_len;
124		if (skip < len)
125			break;
126		skip -= len;
127		n -= len;
128		ix++;
129	}
130
131	while (n && ix < nbv) {
132		len = min3(n, bvecs[ix].bv_len - skip, max_size);
133		span += len;
134		nsegs++;
135		ix++;
136		if (span >= max_size || nsegs >= max_segs)
137			break;
138		skip = 0;
139		n -= len;
140	}
141
142	return min(span, max_size);
143}
144
145/*
146 * Select the span of an xarray iterator we're going to use.  Limit it by both
147 * maximum size and maximum number of segments.  It is assumed that segments
148 * can be larger than a page in size, provided they're physically contiguous.
149 * Returns the size of the span in bytes.
150 */
151static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offset,
152				 size_t max_size, size_t max_segs)
153{
154	struct folio *folio;
155	unsigned int nsegs = 0;
156	loff_t pos = iter->xarray_start + iter->iov_offset;
157	pgoff_t index = pos / PAGE_SIZE;
158	size_t span = 0, n = iter->count;
159
160	XA_STATE(xas, iter->xarray, index);
161
162	if (WARN_ON(!iov_iter_is_xarray(iter)) ||
163	    WARN_ON(start_offset > n) ||
164	    n == 0)
165		return 0;
166	max_size = min(max_size, n - start_offset);
167
168	rcu_read_lock();
169	xas_for_each(&xas, folio, ULONG_MAX) {
170		size_t offset, flen, len;
171		if (xas_retry(&xas, folio))
172			continue;
173		if (WARN_ON(xa_is_value(folio)))
174			break;
175		if (WARN_ON(folio_test_hugetlb(folio)))
176			break;
177
178		flen = folio_size(folio);
179		offset = offset_in_folio(folio, pos);
180		len = min(max_size, flen - offset);
181		span += len;
182		nsegs++;
183		if (span >= max_size || nsegs >= max_segs)
184			break;
185	}
186
187	rcu_read_unlock();
188	return min(span, max_size);
189}
190
191size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
192			size_t max_size, size_t max_segs)
193{
194	if (iov_iter_is_bvec(iter))
195		return netfs_limit_bvec(iter, start_offset, max_size, max_segs);
196	if (iov_iter_is_xarray(iter))
197		return netfs_limit_xarray(iter, start_offset, max_size, max_segs);
198	BUG();
199}
200EXPORT_SYMBOL(netfs_limit_iter);