Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * t10_pi.c - Functions for generating and verifying T10 Protection
4 * Information.
5 */
6
7#include <linux/t10-pi.h>
8#include <linux/blkdev.h>
9#include <linux/crc-t10dif.h>
10#include <net/checksum.h>
11
12typedef __be16 (csum_fn) (void *, unsigned int);
13
14static __be16 t10_pi_crc_fn(void *data, unsigned int len)
15{
16 return cpu_to_be16(crc_t10dif(data, len));
17}
18
19static __be16 t10_pi_ip_fn(void *data, unsigned int len)
20{
21 return (__force __be16)ip_compute_csum(data, len);
22}
23
24/*
25 * Type 1 and Type 2 protection use the same format: 16 bit guard tag,
26 * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
27 * tag.
28 */
29static blk_status_t t10_pi_generate(struct blk_integrity_iter *iter,
30 csum_fn *fn, enum t10_dif_type type)
31{
32 unsigned int i;
33
34 for (i = 0 ; i < iter->data_size ; i += iter->interval) {
35 struct t10_pi_tuple *pi = iter->prot_buf;
36
37 pi->guard_tag = fn(iter->data_buf, iter->interval);
38 pi->app_tag = 0;
39
40 if (type == T10_PI_TYPE1_PROTECTION)
41 pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
42 else
43 pi->ref_tag = 0;
44
45 iter->data_buf += iter->interval;
46 iter->prot_buf += sizeof(struct t10_pi_tuple);
47 iter->seed++;
48 }
49
50 return BLK_STS_OK;
51}
52
53static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
54 csum_fn *fn, enum t10_dif_type type)
55{
56 unsigned int i;
57
58 BUG_ON(type == T10_PI_TYPE0_PROTECTION);
59
60 for (i = 0 ; i < iter->data_size ; i += iter->interval) {
61 struct t10_pi_tuple *pi = iter->prot_buf;
62 __be16 csum;
63
64 if (type == T10_PI_TYPE1_PROTECTION ||
65 type == T10_PI_TYPE2_PROTECTION) {
66 if (pi->app_tag == T10_PI_APP_ESCAPE)
67 goto next;
68
69 if (be32_to_cpu(pi->ref_tag) !=
70 lower_32_bits(iter->seed)) {
71 pr_err("%s: ref tag error at location %llu " \
72 "(rcvd %u)\n", iter->disk_name,
73 (unsigned long long)
74 iter->seed, be32_to_cpu(pi->ref_tag));
75 return BLK_STS_PROTECTION;
76 }
77 } else if (type == T10_PI_TYPE3_PROTECTION) {
78 if (pi->app_tag == T10_PI_APP_ESCAPE &&
79 pi->ref_tag == T10_PI_REF_ESCAPE)
80 goto next;
81 }
82
83 csum = fn(iter->data_buf, iter->interval);
84
85 if (pi->guard_tag != csum) {
86 pr_err("%s: guard tag error at sector %llu " \
87 "(rcvd %04x, want %04x)\n", iter->disk_name,
88 (unsigned long long)iter->seed,
89 be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
90 return BLK_STS_PROTECTION;
91 }
92
93next:
94 iter->data_buf += iter->interval;
95 iter->prot_buf += sizeof(struct t10_pi_tuple);
96 iter->seed++;
97 }
98
99 return BLK_STS_OK;
100}
101
102static blk_status_t t10_pi_type1_generate_crc(struct blk_integrity_iter *iter)
103{
104 return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION);
105}
106
107static blk_status_t t10_pi_type1_generate_ip(struct blk_integrity_iter *iter)
108{
109 return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION);
110}
111
112static blk_status_t t10_pi_type1_verify_crc(struct blk_integrity_iter *iter)
113{
114 return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION);
115}
116
117static blk_status_t t10_pi_type1_verify_ip(struct blk_integrity_iter *iter)
118{
119 return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION);
120}
121
122/**
123 * t10_pi_type1_prepare - prepare PI prior submitting request to device
124 * @rq: request with PI that should be prepared
125 *
126 * For Type 1/Type 2, the virtual start sector is the one that was
127 * originally submitted by the block layer for the ref_tag usage. Due to
128 * partitioning, MD/DM cloning, etc. the actual physical start sector is
129 * likely to be different. Remap protection information to match the
130 * physical LBA.
131 */
132static void t10_pi_type1_prepare(struct request *rq)
133{
134 const int tuple_sz = rq->q->integrity.tuple_size;
135 u32 ref_tag = t10_pi_ref_tag(rq);
136 struct bio *bio;
137
138 __rq_for_each_bio(bio, rq) {
139 struct bio_integrity_payload *bip = bio_integrity(bio);
140 u32 virt = bip_get_seed(bip) & 0xffffffff;
141 struct bio_vec iv;
142 struct bvec_iter iter;
143
144 /* Already remapped? */
145 if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
146 break;
147
148 bip_for_each_vec(iv, bip, iter) {
149 void *p, *pmap;
150 unsigned int j;
151
152 pmap = kmap_atomic(iv.bv_page);
153 p = pmap + iv.bv_offset;
154 for (j = 0; j < iv.bv_len; j += tuple_sz) {
155 struct t10_pi_tuple *pi = p;
156
157 if (be32_to_cpu(pi->ref_tag) == virt)
158 pi->ref_tag = cpu_to_be32(ref_tag);
159 virt++;
160 ref_tag++;
161 p += tuple_sz;
162 }
163
164 kunmap_atomic(pmap);
165 }
166
167 bip->bip_flags |= BIP_MAPPED_INTEGRITY;
168 }
169}
170
171/**
172 * t10_pi_type1_complete - prepare PI prior returning request to the blk layer
173 * @rq: request with PI that should be prepared
174 * @nr_bytes: total bytes to prepare
175 *
176 * For Type 1/Type 2, the virtual start sector is the one that was
177 * originally submitted by the block layer for the ref_tag usage. Due to
178 * partitioning, MD/DM cloning, etc. the actual physical start sector is
179 * likely to be different. Since the physical start sector was submitted
180 * to the device, we should remap it back to virtual values expected by the
181 * block layer.
182 */
183static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
184{
185 unsigned intervals = nr_bytes >> rq->q->integrity.interval_exp;
186 const int tuple_sz = rq->q->integrity.tuple_size;
187 u32 ref_tag = t10_pi_ref_tag(rq);
188 struct bio *bio;
189
190 __rq_for_each_bio(bio, rq) {
191 struct bio_integrity_payload *bip = bio_integrity(bio);
192 u32 virt = bip_get_seed(bip) & 0xffffffff;
193 struct bio_vec iv;
194 struct bvec_iter iter;
195
196 bip_for_each_vec(iv, bip, iter) {
197 void *p, *pmap;
198 unsigned int j;
199
200 pmap = kmap_atomic(iv.bv_page);
201 p = pmap + iv.bv_offset;
202 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
203 struct t10_pi_tuple *pi = p;
204
205 if (be32_to_cpu(pi->ref_tag) == ref_tag)
206 pi->ref_tag = cpu_to_be32(virt);
207 virt++;
208 ref_tag++;
209 intervals--;
210 p += tuple_sz;
211 }
212
213 kunmap_atomic(pmap);
214 }
215 }
216}
217
218static blk_status_t t10_pi_type3_generate_crc(struct blk_integrity_iter *iter)
219{
220 return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION);
221}
222
223static blk_status_t t10_pi_type3_generate_ip(struct blk_integrity_iter *iter)
224{
225 return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
226}
227
228static blk_status_t t10_pi_type3_verify_crc(struct blk_integrity_iter *iter)
229{
230 return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION);
231}
232
233static blk_status_t t10_pi_type3_verify_ip(struct blk_integrity_iter *iter)
234{
235 return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
236}
237
238/**
239 * Type 3 does not have a reference tag so no remapping is required.
240 */
241static void t10_pi_type3_prepare(struct request *rq)
242{
243}
244
245/**
246 * Type 3 does not have a reference tag so no remapping is required.
247 */
248static void t10_pi_type3_complete(struct request *rq, unsigned int nr_bytes)
249{
250}
251
252const struct blk_integrity_profile t10_pi_type1_crc = {
253 .name = "T10-DIF-TYPE1-CRC",
254 .generate_fn = t10_pi_type1_generate_crc,
255 .verify_fn = t10_pi_type1_verify_crc,
256 .prepare_fn = t10_pi_type1_prepare,
257 .complete_fn = t10_pi_type1_complete,
258};
259EXPORT_SYMBOL(t10_pi_type1_crc);
260
261const struct blk_integrity_profile t10_pi_type1_ip = {
262 .name = "T10-DIF-TYPE1-IP",
263 .generate_fn = t10_pi_type1_generate_ip,
264 .verify_fn = t10_pi_type1_verify_ip,
265 .prepare_fn = t10_pi_type1_prepare,
266 .complete_fn = t10_pi_type1_complete,
267};
268EXPORT_SYMBOL(t10_pi_type1_ip);
269
270const struct blk_integrity_profile t10_pi_type3_crc = {
271 .name = "T10-DIF-TYPE3-CRC",
272 .generate_fn = t10_pi_type3_generate_crc,
273 .verify_fn = t10_pi_type3_verify_crc,
274 .prepare_fn = t10_pi_type3_prepare,
275 .complete_fn = t10_pi_type3_complete,
276};
277EXPORT_SYMBOL(t10_pi_type3_crc);
278
279const struct blk_integrity_profile t10_pi_type3_ip = {
280 .name = "T10-DIF-TYPE3-IP",
281 .generate_fn = t10_pi_type3_generate_ip,
282 .verify_fn = t10_pi_type3_verify_ip,
283 .prepare_fn = t10_pi_type3_prepare,
284 .complete_fn = t10_pi_type3_complete,
285};
286EXPORT_SYMBOL(t10_pi_type3_ip);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * t10_pi.c - Functions for generating and verifying T10 Protection
4 * Information.
5 */
6
7#include <linux/t10-pi.h>
8#include <linux/blk-integrity.h>
9#include <linux/crc-t10dif.h>
10#include <linux/crc64.h>
11#include <net/checksum.h>
12#include <linux/unaligned.h>
13#include "blk.h"
14
15struct blk_integrity_iter {
16 void *prot_buf;
17 void *data_buf;
18 sector_t seed;
19 unsigned int data_size;
20 unsigned short interval;
21 const char *disk_name;
22};
23
24static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len,
25 unsigned char csum_type)
26{
27 if (csum_type == BLK_INTEGRITY_CSUM_IP)
28 return (__force __be16)ip_compute_csum(data, len);
29 return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len));
30}
31
32/*
33 * Type 1 and Type 2 protection use the same format: 16 bit guard tag,
34 * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
35 * tag.
36 */
37static void t10_pi_generate(struct blk_integrity_iter *iter,
38 struct blk_integrity *bi)
39{
40 u8 offset = bi->pi_offset;
41 unsigned int i;
42
43 for (i = 0 ; i < iter->data_size ; i += iter->interval) {
44 struct t10_pi_tuple *pi = iter->prot_buf + offset;
45
46 pi->guard_tag = t10_pi_csum(0, iter->data_buf, iter->interval,
47 bi->csum_type);
48 if (offset)
49 pi->guard_tag = t10_pi_csum(pi->guard_tag,
50 iter->prot_buf, offset, bi->csum_type);
51 pi->app_tag = 0;
52
53 if (bi->flags & BLK_INTEGRITY_REF_TAG)
54 pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
55 else
56 pi->ref_tag = 0;
57
58 iter->data_buf += iter->interval;
59 iter->prot_buf += bi->tuple_size;
60 iter->seed++;
61 }
62}
63
64static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
65 struct blk_integrity *bi)
66{
67 u8 offset = bi->pi_offset;
68 unsigned int i;
69
70 for (i = 0 ; i < iter->data_size ; i += iter->interval) {
71 struct t10_pi_tuple *pi = iter->prot_buf + offset;
72 __be16 csum;
73
74 if (bi->flags & BLK_INTEGRITY_REF_TAG) {
75 if (pi->app_tag == T10_PI_APP_ESCAPE)
76 goto next;
77
78 if (be32_to_cpu(pi->ref_tag) !=
79 lower_32_bits(iter->seed)) {
80 pr_err("%s: ref tag error at location %llu " \
81 "(rcvd %u)\n", iter->disk_name,
82 (unsigned long long)
83 iter->seed, be32_to_cpu(pi->ref_tag));
84 return BLK_STS_PROTECTION;
85 }
86 } else {
87 if (pi->app_tag == T10_PI_APP_ESCAPE &&
88 pi->ref_tag == T10_PI_REF_ESCAPE)
89 goto next;
90 }
91
92 csum = t10_pi_csum(0, iter->data_buf, iter->interval,
93 bi->csum_type);
94 if (offset)
95 csum = t10_pi_csum(csum, iter->prot_buf, offset,
96 bi->csum_type);
97
98 if (pi->guard_tag != csum) {
99 pr_err("%s: guard tag error at sector %llu " \
100 "(rcvd %04x, want %04x)\n", iter->disk_name,
101 (unsigned long long)iter->seed,
102 be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
103 return BLK_STS_PROTECTION;
104 }
105
106next:
107 iter->data_buf += iter->interval;
108 iter->prot_buf += bi->tuple_size;
109 iter->seed++;
110 }
111
112 return BLK_STS_OK;
113}
114
115/**
116 * t10_pi_type1_prepare - prepare PI prior submitting request to device
117 * @rq: request with PI that should be prepared
118 *
119 * For Type 1/Type 2, the virtual start sector is the one that was
120 * originally submitted by the block layer for the ref_tag usage. Due to
121 * partitioning, MD/DM cloning, etc. the actual physical start sector is
122 * likely to be different. Remap protection information to match the
123 * physical LBA.
124 */
125static void t10_pi_type1_prepare(struct request *rq)
126{
127 struct blk_integrity *bi = &rq->q->limits.integrity;
128 const int tuple_sz = bi->tuple_size;
129 u32 ref_tag = t10_pi_ref_tag(rq);
130 u8 offset = bi->pi_offset;
131 struct bio *bio;
132
133 __rq_for_each_bio(bio, rq) {
134 struct bio_integrity_payload *bip = bio_integrity(bio);
135 u32 virt = bip_get_seed(bip) & 0xffffffff;
136 struct bio_vec iv;
137 struct bvec_iter iter;
138
139 /* Already remapped? */
140 if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
141 break;
142
143 bip_for_each_vec(iv, bip, iter) {
144 unsigned int j;
145 void *p;
146
147 p = bvec_kmap_local(&iv);
148 for (j = 0; j < iv.bv_len; j += tuple_sz) {
149 struct t10_pi_tuple *pi = p + offset;
150
151 if (be32_to_cpu(pi->ref_tag) == virt)
152 pi->ref_tag = cpu_to_be32(ref_tag);
153 virt++;
154 ref_tag++;
155 p += tuple_sz;
156 }
157 kunmap_local(p);
158 }
159
160 bip->bip_flags |= BIP_MAPPED_INTEGRITY;
161 }
162}
163
164/**
165 * t10_pi_type1_complete - prepare PI prior returning request to the blk layer
166 * @rq: request with PI that should be prepared
167 * @nr_bytes: total bytes to prepare
168 *
169 * For Type 1/Type 2, the virtual start sector is the one that was
170 * originally submitted by the block layer for the ref_tag usage. Due to
171 * partitioning, MD/DM cloning, etc. the actual physical start sector is
172 * likely to be different. Since the physical start sector was submitted
173 * to the device, we should remap it back to virtual values expected by the
174 * block layer.
175 */
176static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
177{
178 struct blk_integrity *bi = &rq->q->limits.integrity;
179 unsigned intervals = nr_bytes >> bi->interval_exp;
180 const int tuple_sz = bi->tuple_size;
181 u32 ref_tag = t10_pi_ref_tag(rq);
182 u8 offset = bi->pi_offset;
183 struct bio *bio;
184
185 __rq_for_each_bio(bio, rq) {
186 struct bio_integrity_payload *bip = bio_integrity(bio);
187 u32 virt = bip_get_seed(bip) & 0xffffffff;
188 struct bio_vec iv;
189 struct bvec_iter iter;
190
191 bip_for_each_vec(iv, bip, iter) {
192 unsigned int j;
193 void *p;
194
195 p = bvec_kmap_local(&iv);
196 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
197 struct t10_pi_tuple *pi = p + offset;
198
199 if (be32_to_cpu(pi->ref_tag) == ref_tag)
200 pi->ref_tag = cpu_to_be32(virt);
201 virt++;
202 ref_tag++;
203 intervals--;
204 p += tuple_sz;
205 }
206 kunmap_local(p);
207 }
208 }
209}
210
211static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)
212{
213 return cpu_to_be64(crc64_rocksoft_update(crc, data, len));
214}
215
216static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,
217 struct blk_integrity *bi)
218{
219 u8 offset = bi->pi_offset;
220 unsigned int i;
221
222 for (i = 0 ; i < iter->data_size ; i += iter->interval) {
223 struct crc64_pi_tuple *pi = iter->prot_buf + offset;
224
225 pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval);
226 if (offset)
227 pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag),
228 iter->prot_buf, offset);
229 pi->app_tag = 0;
230
231 if (bi->flags & BLK_INTEGRITY_REF_TAG)
232 put_unaligned_be48(iter->seed, pi->ref_tag);
233 else
234 put_unaligned_be48(0ULL, pi->ref_tag);
235
236 iter->data_buf += iter->interval;
237 iter->prot_buf += bi->tuple_size;
238 iter->seed++;
239 }
240}
241
242static bool ext_pi_ref_escape(const u8 ref_tag[6])
243{
244 static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
245
246 return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;
247}
248
249static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,
250 struct blk_integrity *bi)
251{
252 u8 offset = bi->pi_offset;
253 unsigned int i;
254
255 for (i = 0; i < iter->data_size; i += iter->interval) {
256 struct crc64_pi_tuple *pi = iter->prot_buf + offset;
257 u64 ref, seed;
258 __be64 csum;
259
260 if (bi->flags & BLK_INTEGRITY_REF_TAG) {
261 if (pi->app_tag == T10_PI_APP_ESCAPE)
262 goto next;
263
264 ref = get_unaligned_be48(pi->ref_tag);
265 seed = lower_48_bits(iter->seed);
266 if (ref != seed) {
267 pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
268 iter->disk_name, seed, ref);
269 return BLK_STS_PROTECTION;
270 }
271 } else {
272 if (pi->app_tag == T10_PI_APP_ESCAPE &&
273 ext_pi_ref_escape(pi->ref_tag))
274 goto next;
275 }
276
277 csum = ext_pi_crc64(0, iter->data_buf, iter->interval);
278 if (offset)
279 csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf,
280 offset);
281
282 if (pi->guard_tag != csum) {
283 pr_err("%s: guard tag error at sector %llu " \
284 "(rcvd %016llx, want %016llx)\n",
285 iter->disk_name, (unsigned long long)iter->seed,
286 be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));
287 return BLK_STS_PROTECTION;
288 }
289
290next:
291 iter->data_buf += iter->interval;
292 iter->prot_buf += bi->tuple_size;
293 iter->seed++;
294 }
295
296 return BLK_STS_OK;
297}
298
299static void ext_pi_type1_prepare(struct request *rq)
300{
301 struct blk_integrity *bi = &rq->q->limits.integrity;
302 const int tuple_sz = bi->tuple_size;
303 u64 ref_tag = ext_pi_ref_tag(rq);
304 u8 offset = bi->pi_offset;
305 struct bio *bio;
306
307 __rq_for_each_bio(bio, rq) {
308 struct bio_integrity_payload *bip = bio_integrity(bio);
309 u64 virt = lower_48_bits(bip_get_seed(bip));
310 struct bio_vec iv;
311 struct bvec_iter iter;
312
313 /* Already remapped? */
314 if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
315 break;
316
317 bip_for_each_vec(iv, bip, iter) {
318 unsigned int j;
319 void *p;
320
321 p = bvec_kmap_local(&iv);
322 for (j = 0; j < iv.bv_len; j += tuple_sz) {
323 struct crc64_pi_tuple *pi = p + offset;
324 u64 ref = get_unaligned_be48(pi->ref_tag);
325
326 if (ref == virt)
327 put_unaligned_be48(ref_tag, pi->ref_tag);
328 virt++;
329 ref_tag++;
330 p += tuple_sz;
331 }
332 kunmap_local(p);
333 }
334
335 bip->bip_flags |= BIP_MAPPED_INTEGRITY;
336 }
337}
338
339static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
340{
341 struct blk_integrity *bi = &rq->q->limits.integrity;
342 unsigned intervals = nr_bytes >> bi->interval_exp;
343 const int tuple_sz = bi->tuple_size;
344 u64 ref_tag = ext_pi_ref_tag(rq);
345 u8 offset = bi->pi_offset;
346 struct bio *bio;
347
348 __rq_for_each_bio(bio, rq) {
349 struct bio_integrity_payload *bip = bio_integrity(bio);
350 u64 virt = lower_48_bits(bip_get_seed(bip));
351 struct bio_vec iv;
352 struct bvec_iter iter;
353
354 bip_for_each_vec(iv, bip, iter) {
355 unsigned int j;
356 void *p;
357
358 p = bvec_kmap_local(&iv);
359 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
360 struct crc64_pi_tuple *pi = p + offset;
361 u64 ref = get_unaligned_be48(pi->ref_tag);
362
363 if (ref == ref_tag)
364 put_unaligned_be48(virt, pi->ref_tag);
365 virt++;
366 ref_tag++;
367 intervals--;
368 p += tuple_sz;
369 }
370 kunmap_local(p);
371 }
372 }
373}
374
375void blk_integrity_generate(struct bio *bio)
376{
377 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
378 struct bio_integrity_payload *bip = bio_integrity(bio);
379 struct blk_integrity_iter iter;
380 struct bvec_iter bviter;
381 struct bio_vec bv;
382
383 iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
384 iter.interval = 1 << bi->interval_exp;
385 iter.seed = bio->bi_iter.bi_sector;
386 iter.prot_buf = bvec_virt(bip->bip_vec);
387 bio_for_each_segment(bv, bio, bviter) {
388 void *kaddr = bvec_kmap_local(&bv);
389
390 iter.data_buf = kaddr;
391 iter.data_size = bv.bv_len;
392 switch (bi->csum_type) {
393 case BLK_INTEGRITY_CSUM_CRC64:
394 ext_pi_crc64_generate(&iter, bi);
395 break;
396 case BLK_INTEGRITY_CSUM_CRC:
397 case BLK_INTEGRITY_CSUM_IP:
398 t10_pi_generate(&iter, bi);
399 break;
400 default:
401 break;
402 }
403 kunmap_local(kaddr);
404 }
405}
406
407void blk_integrity_verify(struct bio *bio)
408{
409 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
410 struct bio_integrity_payload *bip = bio_integrity(bio);
411 struct blk_integrity_iter iter;
412 struct bvec_iter bviter;
413 struct bio_vec bv;
414
415 /*
416 * At the moment verify is called bi_iter has been advanced during split
417 * and completion, so use the copy created during submission here.
418 */
419 iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
420 iter.interval = 1 << bi->interval_exp;
421 iter.seed = bip->bio_iter.bi_sector;
422 iter.prot_buf = bvec_virt(bip->bip_vec);
423 __bio_for_each_segment(bv, bio, bviter, bip->bio_iter) {
424 void *kaddr = bvec_kmap_local(&bv);
425 blk_status_t ret = BLK_STS_OK;
426
427 iter.data_buf = kaddr;
428 iter.data_size = bv.bv_len;
429 switch (bi->csum_type) {
430 case BLK_INTEGRITY_CSUM_CRC64:
431 ret = ext_pi_crc64_verify(&iter, bi);
432 break;
433 case BLK_INTEGRITY_CSUM_CRC:
434 case BLK_INTEGRITY_CSUM_IP:
435 ret = t10_pi_verify(&iter, bi);
436 break;
437 default:
438 break;
439 }
440 kunmap_local(kaddr);
441
442 if (ret) {
443 bio->bi_status = ret;
444 return;
445 }
446 }
447}
448
449void blk_integrity_prepare(struct request *rq)
450{
451 struct blk_integrity *bi = &rq->q->limits.integrity;
452
453 if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
454 return;
455
456 if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
457 ext_pi_type1_prepare(rq);
458 else
459 t10_pi_type1_prepare(rq);
460}
461
462void blk_integrity_complete(struct request *rq, unsigned int nr_bytes)
463{
464 struct blk_integrity *bi = &rq->q->limits.integrity;
465
466 if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
467 return;
468
469 if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
470 ext_pi_type1_complete(rq, nr_bytes);
471 else
472 t10_pi_type1_complete(rq, nr_bytes);
473}