1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Copyright 2022 Red Hat, Inc.
  4 */
  5
  6#include <linux/bio.h>
  7#include <linux/blk-crypto.h>
  8#include <linux/blk-integrity.h>
  9
 10#include "dm-core.h"
 11
 12static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv,
 13				       struct bvec_iter *iter,
 14				       unsigned int bytes)
 15{
 16	int idx;
 17
 18	iter->bi_size += bytes;
 19	if (bytes <= iter->bi_bvec_done) {
 20		iter->bi_bvec_done -= bytes;
 21		return true;
 22	}
 23
 24	bytes -= iter->bi_bvec_done;
 25	idx = iter->bi_idx - 1;
 26
 27	while (idx >= 0 && bytes && bytes > bv[idx].bv_len) {
 28		bytes -= bv[idx].bv_len;
 29		idx--;
 30	}
 31
 32	if (WARN_ONCE(idx < 0 && bytes,
 33		      "Attempted to rewind iter beyond bvec's boundaries\n")) {
 34		iter->bi_size -= bytes;
 35		iter->bi_bvec_done = 0;
 36		iter->bi_idx = 0;
 37		return false;
 38	}
 39
 40	iter->bi_idx = idx;
 41	iter->bi_bvec_done = bv[idx].bv_len - bytes;
 42	return true;
 43}
 44
 45#if defined(CONFIG_BLK_DEV_INTEGRITY)
 46
 47/**
 48 * dm_bio_integrity_rewind - Rewind integrity vector
 49 * @bio:	bio whose integrity vector to update
 50 * @bytes_done:	number of data bytes to rewind
 51 *
 52 * Description: This function calculates how many integrity bytes the
 53 * number of completed data bytes correspond to and rewind the
 54 * integrity vector accordingly.
 55 */
 56static void dm_bio_integrity_rewind(struct bio *bio, unsigned int bytes_done)
 57{
 58	struct bio_integrity_payload *bip = bio_integrity(bio);
 59	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
 60	unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
 61
 62	bip->bip_iter.bi_sector -= bio_integrity_intervals(bi, bytes_done >> 9);
 63	dm_bvec_iter_rewind(bip->bip_vec, &bip->bip_iter, bytes);
 64}
 65
 66#else /* CONFIG_BLK_DEV_INTEGRITY */
 67
 68static inline void dm_bio_integrity_rewind(struct bio *bio,
 69					   unsigned int bytes_done)
 70{
 71	return;
 72}
 73
 74#endif
 75
 76#if defined(CONFIG_BLK_INLINE_ENCRYPTION)
 77
 78/* Decrements @dun by @dec, treating @dun as a multi-limb integer. */
 79static void dm_bio_crypt_dun_decrement(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
 80				       unsigned int dec)
 81{
 82	int i;
 83
 84	for (i = 0; dec && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) {
 85		u64 prev = dun[i];
 86
 87		dun[i] -= dec;
 88		if (dun[i] > prev)
 89			dec = 1;
 90		else
 91			dec = 0;
 92	}
 93}
 94
 95static void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
 96{
 97	struct bio_crypt_ctx *bc = bio->bi_crypt_context;
 98
 99	dm_bio_crypt_dun_decrement(bc->bc_dun,
100				   bytes >> bc->bc_key->data_unit_size_bits);
101}
102
103#else /* CONFIG_BLK_INLINE_ENCRYPTION */
104
105static inline void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
106{
107	return;
108}
109
110#endif
111
112static inline void dm_bio_rewind_iter(const struct bio *bio,
113				      struct bvec_iter *iter, unsigned int bytes)
114{
115	iter->bi_sector -= bytes >> 9;
116
117	/* No advance means no rewind */
118	if (bio_no_advance_iter(bio))
119		iter->bi_size += bytes;
120	else
121		dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes);
122}
123
124/**
125 * dm_bio_rewind - update ->bi_iter of @bio by rewinding @bytes.
126 * @bio: bio to rewind
127 * @bytes: how many bytes to rewind
128 *
129 * WARNING:
130 * Caller must ensure that @bio has a fixed end sector, to allow
131 * rewinding from end of bio and restoring its original position.
132 * Caller is also responsibile for restoring bio's size.
133 */
134static void dm_bio_rewind(struct bio *bio, unsigned bytes)
135{
136	if (bio_integrity(bio))
137		dm_bio_integrity_rewind(bio, bytes);
138
139	if (bio_has_crypt_ctx(bio))
140		dm_bio_crypt_rewind(bio, bytes);
141
142	dm_bio_rewind_iter(bio, &bio->bi_iter, bytes);
143}
144
145void dm_io_rewind(struct dm_io *io, struct bio_set *bs)
146{
147	struct bio *orig = io->orig_bio;
148	struct bio *new_orig = bio_alloc_clone(orig->bi_bdev, orig,
149					       GFP_NOIO, bs);
150	/*
151	 * dm_bio_rewind can restore to previous position since the
152	 * end sector is fixed for original bio, but we still need
153	 * to restore bio's size manually (using io->sectors).
154	 */
155	dm_bio_rewind(new_orig, ((io->sector_offset << 9) -
156				 orig->bi_iter.bi_size));
157	bio_trim(new_orig, 0, io->sectors);
158
159	bio_chain(new_orig, orig);
160	/*
161	 * __bi_remaining was increased (by dm_split_and_process_bio),
162	 * so must drop the one added in bio_chain.
163	 */
164	atomic_dec(&orig->__bi_remaining);
165	io->orig_bio = new_orig;
166}