Loading...
Note: File does not exist in v3.1.
1/*
2 * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved.
3 * Copyright (C) 2016-2017 Milan Broz
4 * Copyright (C) 2016-2017 Mikulas Patocka
5 *
6 * This file is released under the GPL.
7 */
8
9#include "dm-bio-record.h"
10
11#include <linux/compiler.h>
12#include <linux/module.h>
13#include <linux/device-mapper.h>
14#include <linux/dm-io.h>
15#include <linux/vmalloc.h>
16#include <linux/sort.h>
17#include <linux/rbtree.h>
18#include <linux/delay.h>
19#include <linux/random.h>
20#include <linux/reboot.h>
21#include <crypto/hash.h>
22#include <crypto/skcipher.h>
23#include <linux/async_tx.h>
24#include <linux/dm-bufio.h>
25
26#include "dm-audit.h"
27
28#define DM_MSG_PREFIX "integrity"
29
30#define DEFAULT_INTERLEAVE_SECTORS 32768
31#define DEFAULT_JOURNAL_SIZE_FACTOR 7
32#define DEFAULT_SECTORS_PER_BITMAP_BIT 32768
33#define DEFAULT_BUFFER_SECTORS 128
34#define DEFAULT_JOURNAL_WATERMARK 50
35#define DEFAULT_SYNC_MSEC 10000
36#define DEFAULT_MAX_JOURNAL_SECTORS 131072
37#define MIN_LOG2_INTERLEAVE_SECTORS 3
38#define MAX_LOG2_INTERLEAVE_SECTORS 31
39#define METADATA_WORKQUEUE_MAX_ACTIVE 16
40#define RECALC_SECTORS 32768
41#define RECALC_WRITE_SUPER 16
42#define BITMAP_BLOCK_SIZE 4096 /* don't change it */
43#define BITMAP_FLUSH_INTERVAL (10 * HZ)
44#define DISCARD_FILLER 0xf6
45#define SALT_SIZE 16
46
47/*
48 * Warning - DEBUG_PRINT prints security-sensitive data to the log,
49 * so it should not be enabled in the official kernel
50 */
51//#define DEBUG_PRINT
52//#define INTERNAL_VERIFY
53
54/*
55 * On disk structures
56 */
57
58#define SB_MAGIC "integrt"
59#define SB_VERSION_1 1
60#define SB_VERSION_2 2
61#define SB_VERSION_3 3
62#define SB_VERSION_4 4
63#define SB_VERSION_5 5
64#define SB_SECTORS 8
65#define MAX_SECTORS_PER_BLOCK 8
66
67struct superblock {
68 __u8 magic[8];
69 __u8 version;
70 __u8 log2_interleave_sectors;
71 __le16 integrity_tag_size;
72 __le32 journal_sections;
73 __le64 provided_data_sectors; /* userspace uses this value */
74 __le32 flags;
75 __u8 log2_sectors_per_block;
76 __u8 log2_blocks_per_bitmap_bit;
77 __u8 pad[2];
78 __le64 recalc_sector;
79 __u8 pad2[8];
80 __u8 salt[SALT_SIZE];
81};
82
83#define SB_FLAG_HAVE_JOURNAL_MAC 0x1
84#define SB_FLAG_RECALCULATING 0x2
85#define SB_FLAG_DIRTY_BITMAP 0x4
86#define SB_FLAG_FIXED_PADDING 0x8
87#define SB_FLAG_FIXED_HMAC 0x10
88
89#define JOURNAL_ENTRY_ROUNDUP 8
90
91typedef __le64 commit_id_t;
92#define JOURNAL_MAC_PER_SECTOR 8
93
94struct journal_entry {
95 union {
96 struct {
97 __le32 sector_lo;
98 __le32 sector_hi;
99 } s;
100 __le64 sector;
101 } u;
102 commit_id_t last_bytes[];
103 /* __u8 tag[0]; */
104};
105
106#define journal_entry_tag(ic, je) ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block])
107
108#if BITS_PER_LONG == 64
109#define journal_entry_set_sector(je, x) do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0)
110#else
111#define journal_entry_set_sector(je, x) do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0)
112#endif
113#define journal_entry_get_sector(je) le64_to_cpu((je)->u.sector)
114#define journal_entry_is_unused(je) ((je)->u.s.sector_hi == cpu_to_le32(-1))
115#define journal_entry_set_unused(je) do { ((je)->u.s.sector_hi = cpu_to_le32(-1)); } while (0)
116#define journal_entry_is_inprogress(je) ((je)->u.s.sector_hi == cpu_to_le32(-2))
117#define journal_entry_set_inprogress(je) do { ((je)->u.s.sector_hi = cpu_to_le32(-2)); } while (0)
118
119#define JOURNAL_BLOCK_SECTORS 8
120#define JOURNAL_SECTOR_DATA ((1 << SECTOR_SHIFT) - sizeof(commit_id_t))
121#define JOURNAL_MAC_SIZE (JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS)
122
123struct journal_sector {
124 struct_group(sectors,
125 __u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR];
126 __u8 mac[JOURNAL_MAC_PER_SECTOR];
127 );
128 commit_id_t commit_id;
129};
130
131#define MAX_TAG_SIZE (JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK]))
132
133#define METADATA_PADDING_SECTORS 8
134
135#define N_COMMIT_IDS 4
136
137static unsigned char prev_commit_seq(unsigned char seq)
138{
139 return (seq + N_COMMIT_IDS - 1) % N_COMMIT_IDS;
140}
141
142static unsigned char next_commit_seq(unsigned char seq)
143{
144 return (seq + 1) % N_COMMIT_IDS;
145}
146
147/*
148 * In-memory structures
149 */
150
151struct journal_node {
152 struct rb_node node;
153 sector_t sector;
154};
155
156struct alg_spec {
157 char *alg_string;
158 char *key_string;
159 __u8 *key;
160 unsigned key_size;
161};
162
163struct dm_integrity_c {
164 struct dm_dev *dev;
165 struct dm_dev *meta_dev;
166 unsigned tag_size;
167 __s8 log2_tag_size;
168 sector_t start;
169 mempool_t journal_io_mempool;
170 struct dm_io_client *io;
171 struct dm_bufio_client *bufio;
172 struct workqueue_struct *metadata_wq;
173 struct superblock *sb;
174 unsigned journal_pages;
175 unsigned n_bitmap_blocks;
176
177 struct page_list *journal;
178 struct page_list *journal_io;
179 struct page_list *journal_xor;
180 struct page_list *recalc_bitmap;
181 struct page_list *may_write_bitmap;
182 struct bitmap_block_status *bbs;
183 unsigned bitmap_flush_interval;
184 int synchronous_mode;
185 struct bio_list synchronous_bios;
186 struct delayed_work bitmap_flush_work;
187
188 struct crypto_skcipher *journal_crypt;
189 struct scatterlist **journal_scatterlist;
190 struct scatterlist **journal_io_scatterlist;
191 struct skcipher_request **sk_requests;
192
193 struct crypto_shash *journal_mac;
194
195 struct journal_node *journal_tree;
196 struct rb_root journal_tree_root;
197
198 sector_t provided_data_sectors;
199
200 unsigned short journal_entry_size;
201 unsigned char journal_entries_per_sector;
202 unsigned char journal_section_entries;
203 unsigned short journal_section_sectors;
204 unsigned journal_sections;
205 unsigned journal_entries;
206 sector_t data_device_sectors;
207 sector_t meta_device_sectors;
208 unsigned initial_sectors;
209 unsigned metadata_run;
210 __s8 log2_metadata_run;
211 __u8 log2_buffer_sectors;
212 __u8 sectors_per_block;
213 __u8 log2_blocks_per_bitmap_bit;
214
215 unsigned char mode;
216
217 int failed;
218
219 struct crypto_shash *internal_hash;
220
221 struct dm_target *ti;
222
223 /* these variables are locked with endio_wait.lock */
224 struct rb_root in_progress;
225 struct list_head wait_list;
226 wait_queue_head_t endio_wait;
227 struct workqueue_struct *wait_wq;
228 struct workqueue_struct *offload_wq;
229
230 unsigned char commit_seq;
231 commit_id_t commit_ids[N_COMMIT_IDS];
232
233 unsigned committed_section;
234 unsigned n_committed_sections;
235
236 unsigned uncommitted_section;
237 unsigned n_uncommitted_sections;
238
239 unsigned free_section;
240 unsigned char free_section_entry;
241 unsigned free_sectors;
242
243 unsigned free_sectors_threshold;
244
245 struct workqueue_struct *commit_wq;
246 struct work_struct commit_work;
247
248 struct workqueue_struct *writer_wq;
249 struct work_struct writer_work;
250
251 struct workqueue_struct *recalc_wq;
252 struct work_struct recalc_work;
253 u8 *recalc_buffer;
254 u8 *recalc_tags;
255
256 struct bio_list flush_bio_list;
257
258 unsigned long autocommit_jiffies;
259 struct timer_list autocommit_timer;
260 unsigned autocommit_msec;
261
262 wait_queue_head_t copy_to_journal_wait;
263
264 struct completion crypto_backoff;
265
266 bool wrote_to_journal;
267 bool journal_uptodate;
268 bool just_formatted;
269 bool recalculate_flag;
270 bool reset_recalculate_flag;
271 bool discard;
272 bool fix_padding;
273 bool fix_hmac;
274 bool legacy_recalculate;
275
276 struct alg_spec internal_hash_alg;
277 struct alg_spec journal_crypt_alg;
278 struct alg_spec journal_mac_alg;
279
280 atomic64_t number_of_mismatches;
281
282 struct notifier_block reboot_notifier;
283};
284
285struct dm_integrity_range {
286 sector_t logical_sector;
287 sector_t n_sectors;
288 bool waiting;
289 union {
290 struct rb_node node;
291 struct {
292 struct task_struct *task;
293 struct list_head wait_entry;
294 };
295 };
296};
297
298struct dm_integrity_io {
299 struct work_struct work;
300
301 struct dm_integrity_c *ic;
302 enum req_op op;
303 bool fua;
304
305 struct dm_integrity_range range;
306
307 sector_t metadata_block;
308 unsigned metadata_offset;
309
310 atomic_t in_flight;
311 blk_status_t bi_status;
312
313 struct completion *completion;
314
315 struct dm_bio_details bio_details;
316};
317
318struct journal_completion {
319 struct dm_integrity_c *ic;
320 atomic_t in_flight;
321 struct completion comp;
322};
323
324struct journal_io {
325 struct dm_integrity_range range;
326 struct journal_completion *comp;
327};
328
329struct bitmap_block_status {
330 struct work_struct work;
331 struct dm_integrity_c *ic;
332 unsigned idx;
333 unsigned long *bitmap;
334 struct bio_list bio_queue;
335 spinlock_t bio_queue_lock;
336
337};
338
339static struct kmem_cache *journal_io_cache;
340
341#define JOURNAL_IO_MEMPOOL 32
342
343#ifdef DEBUG_PRINT
344#define DEBUG_print(x, ...) printk(KERN_DEBUG x, ##__VA_ARGS__)
345static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...)
346{
347 va_list args;
348 va_start(args, msg);
349 vprintk(msg, args);
350 va_end(args);
351 if (len)
352 pr_cont(":");
353 while (len) {
354 pr_cont(" %02x", *bytes);
355 bytes++;
356 len--;
357 }
358 pr_cont("\n");
359}
360#define DEBUG_bytes(bytes, len, msg, ...) __DEBUG_bytes(bytes, len, KERN_DEBUG msg, ##__VA_ARGS__)
361#else
362#define DEBUG_print(x, ...) do { } while (0)
363#define DEBUG_bytes(bytes, len, msg, ...) do { } while (0)
364#endif
365
366static void dm_integrity_prepare(struct request *rq)
367{
368}
369
370static void dm_integrity_complete(struct request *rq, unsigned int nr_bytes)
371{
372}
373
374/*
375 * DM Integrity profile, protection is performed layer above (dm-crypt)
376 */
377static const struct blk_integrity_profile dm_integrity_profile = {
378 .name = "DM-DIF-EXT-TAG",
379 .generate_fn = NULL,
380 .verify_fn = NULL,
381 .prepare_fn = dm_integrity_prepare,
382 .complete_fn = dm_integrity_complete,
383};
384
385static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
386static void integrity_bio_wait(struct work_struct *w);
387static void dm_integrity_dtr(struct dm_target *ti);
388
389static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err)
390{
391 if (err == -EILSEQ)
392 atomic64_inc(&ic->number_of_mismatches);
393 if (!cmpxchg(&ic->failed, 0, err))
394 DMERR("Error on %s: %d", msg, err);
395}
396
397static int dm_integrity_failed(struct dm_integrity_c *ic)
398{
399 return READ_ONCE(ic->failed);
400}
401
402static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic)
403{
404 if (ic->legacy_recalculate)
405 return false;
406 if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ?
407 ic->internal_hash_alg.key || ic->journal_mac_alg.key :
408 ic->internal_hash_alg.key && !ic->journal_mac_alg.key)
409 return true;
410 return false;
411}
412
413static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i,
414 unsigned j, unsigned char seq)
415{
416 /*
417 * Xor the number with section and sector, so that if a piece of
418 * journal is written at wrong place, it is detected.
419 */
420 return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j);
421}
422
423static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector,
424 sector_t *area, sector_t *offset)
425{
426 if (!ic->meta_dev) {
427 __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors;
428 *area = data_sector >> log2_interleave_sectors;
429 *offset = (unsigned)data_sector & ((1U << log2_interleave_sectors) - 1);
430 } else {
431 *area = 0;
432 *offset = data_sector;
433 }
434}
435
436#define sector_to_block(ic, n) \
437do { \
438 BUG_ON((n) & (unsigned)((ic)->sectors_per_block - 1)); \
439 (n) >>= (ic)->sb->log2_sectors_per_block; \
440} while (0)
441
442static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area,
443 sector_t offset, unsigned *metadata_offset)
444{
445 __u64 ms;
446 unsigned mo;
447
448 ms = area << ic->sb->log2_interleave_sectors;
449 if (likely(ic->log2_metadata_run >= 0))
450 ms += area << ic->log2_metadata_run;
451 else
452 ms += area * ic->metadata_run;
453 ms >>= ic->log2_buffer_sectors;
454
455 sector_to_block(ic, offset);
456
457 if (likely(ic->log2_tag_size >= 0)) {
458 ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size);
459 mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
460 } else {
461 ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors);
462 mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
463 }
464 *metadata_offset = mo;
465 return ms;
466}
467
468static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset)
469{
470 sector_t result;
471
472 if (ic->meta_dev)
473 return offset;
474
475 result = area << ic->sb->log2_interleave_sectors;
476 if (likely(ic->log2_metadata_run >= 0))
477 result += (area + 1) << ic->log2_metadata_run;
478 else
479 result += (area + 1) * ic->metadata_run;
480
481 result += (sector_t)ic->initial_sectors + offset;
482 result += ic->start;
483
484 return result;
485}
486
487static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr)
488{
489 if (unlikely(*sec_ptr >= ic->journal_sections))
490 *sec_ptr -= ic->journal_sections;
491}
492
493static void sb_set_version(struct dm_integrity_c *ic)
494{
495 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC))
496 ic->sb->version = SB_VERSION_5;
497 else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING))
498 ic->sb->version = SB_VERSION_4;
499 else if (ic->mode == 'B' || ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP))
500 ic->sb->version = SB_VERSION_3;
501 else if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
502 ic->sb->version = SB_VERSION_2;
503 else
504 ic->sb->version = SB_VERSION_1;
505}
506
507static int sb_mac(struct dm_integrity_c *ic, bool wr)
508{
509 SHASH_DESC_ON_STACK(desc, ic->journal_mac);
510 int r;
511 unsigned size = crypto_shash_digestsize(ic->journal_mac);
512
513 if (sizeof(struct superblock) + size > 1 << SECTOR_SHIFT) {
514 dm_integrity_io_error(ic, "digest is too long", -EINVAL);
515 return -EINVAL;
516 }
517
518 desc->tfm = ic->journal_mac;
519
520 r = crypto_shash_init(desc);
521 if (unlikely(r < 0)) {
522 dm_integrity_io_error(ic, "crypto_shash_init", r);
523 return r;
524 }
525
526 r = crypto_shash_update(desc, (__u8 *)ic->sb, (1 << SECTOR_SHIFT) - size);
527 if (unlikely(r < 0)) {
528 dm_integrity_io_error(ic, "crypto_shash_update", r);
529 return r;
530 }
531
532 if (likely(wr)) {
533 r = crypto_shash_final(desc, (__u8 *)ic->sb + (1 << SECTOR_SHIFT) - size);
534 if (unlikely(r < 0)) {
535 dm_integrity_io_error(ic, "crypto_shash_final", r);
536 return r;
537 }
538 } else {
539 __u8 result[HASH_MAX_DIGESTSIZE];
540 r = crypto_shash_final(desc, result);
541 if (unlikely(r < 0)) {
542 dm_integrity_io_error(ic, "crypto_shash_final", r);
543 return r;
544 }
545 if (memcmp((__u8 *)ic->sb + (1 << SECTOR_SHIFT) - size, result, size)) {
546 dm_integrity_io_error(ic, "superblock mac", -EILSEQ);
547 dm_audit_log_target(DM_MSG_PREFIX, "mac-superblock", ic->ti, 0);
548 return -EILSEQ;
549 }
550 }
551
552 return 0;
553}
554
555static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf)
556{
557 struct dm_io_request io_req;
558 struct dm_io_region io_loc;
559 const enum req_op op = opf & REQ_OP_MASK;
560 int r;
561
562 io_req.bi_opf = opf;
563 io_req.mem.type = DM_IO_KMEM;
564 io_req.mem.ptr.addr = ic->sb;
565 io_req.notify.fn = NULL;
566 io_req.client = ic->io;
567 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
568 io_loc.sector = ic->start;
569 io_loc.count = SB_SECTORS;
570
571 if (op == REQ_OP_WRITE) {
572 sb_set_version(ic);
573 if (ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
574 r = sb_mac(ic, true);
575 if (unlikely(r))
576 return r;
577 }
578 }
579
580 r = dm_io(&io_req, 1, &io_loc, NULL);
581 if (unlikely(r))
582 return r;
583
584 if (op == REQ_OP_READ) {
585 if (ic->mode != 'R' && ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
586 r = sb_mac(ic, false);
587 if (unlikely(r))
588 return r;
589 }
590 }
591
592 return 0;
593}
594
595#define BITMAP_OP_TEST_ALL_SET 0
596#define BITMAP_OP_TEST_ALL_CLEAR 1
597#define BITMAP_OP_SET 2
598#define BITMAP_OP_CLEAR 3
599
600static bool block_bitmap_op(struct dm_integrity_c *ic, struct page_list *bitmap,
601 sector_t sector, sector_t n_sectors, int mode)
602{
603 unsigned long bit, end_bit, this_end_bit, page, end_page;
604 unsigned long *data;
605
606 if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) {
607 DMCRIT("invalid bitmap access (%llx,%llx,%d,%d,%d)",
608 sector,
609 n_sectors,
610 ic->sb->log2_sectors_per_block,
611 ic->log2_blocks_per_bitmap_bit,
612 mode);
613 BUG();
614 }
615
616 if (unlikely(!n_sectors))
617 return true;
618
619 bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
620 end_bit = (sector + n_sectors - 1) >>
621 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
622
623 page = bit / (PAGE_SIZE * 8);
624 bit %= PAGE_SIZE * 8;
625
626 end_page = end_bit / (PAGE_SIZE * 8);
627 end_bit %= PAGE_SIZE * 8;
628
629repeat:
630 if (page < end_page) {
631 this_end_bit = PAGE_SIZE * 8 - 1;
632 } else {
633 this_end_bit = end_bit;
634 }
635
636 data = lowmem_page_address(bitmap[page].page);
637
638 if (mode == BITMAP_OP_TEST_ALL_SET) {
639 while (bit <= this_end_bit) {
640 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
641 do {
642 if (data[bit / BITS_PER_LONG] != -1)
643 return false;
644 bit += BITS_PER_LONG;
645 } while (this_end_bit >= bit + BITS_PER_LONG - 1);
646 continue;
647 }
648 if (!test_bit(bit, data))
649 return false;
650 bit++;
651 }
652 } else if (mode == BITMAP_OP_TEST_ALL_CLEAR) {
653 while (bit <= this_end_bit) {
654 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
655 do {
656 if (data[bit / BITS_PER_LONG] != 0)
657 return false;
658 bit += BITS_PER_LONG;
659 } while (this_end_bit >= bit + BITS_PER_LONG - 1);
660 continue;
661 }
662 if (test_bit(bit, data))
663 return false;
664 bit++;
665 }
666 } else if (mode == BITMAP_OP_SET) {
667 while (bit <= this_end_bit) {
668 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
669 do {
670 data[bit / BITS_PER_LONG] = -1;
671 bit += BITS_PER_LONG;
672 } while (this_end_bit >= bit + BITS_PER_LONG - 1);
673 continue;
674 }
675 __set_bit(bit, data);
676 bit++;
677 }
678 } else if (mode == BITMAP_OP_CLEAR) {
679 if (!bit && this_end_bit == PAGE_SIZE * 8 - 1)
680 clear_page(data);
681 else while (bit <= this_end_bit) {
682 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
683 do {
684 data[bit / BITS_PER_LONG] = 0;
685 bit += BITS_PER_LONG;
686 } while (this_end_bit >= bit + BITS_PER_LONG - 1);
687 continue;
688 }
689 __clear_bit(bit, data);
690 bit++;
691 }
692 } else {
693 BUG();
694 }
695
696 if (unlikely(page < end_page)) {
697 bit = 0;
698 page++;
699 goto repeat;
700 }
701
702 return true;
703}
704
705static void block_bitmap_copy(struct dm_integrity_c *ic, struct page_list *dst, struct page_list *src)
706{
707 unsigned n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE);
708 unsigned i;
709
710 for (i = 0; i < n_bitmap_pages; i++) {
711 unsigned long *dst_data = lowmem_page_address(dst[i].page);
712 unsigned long *src_data = lowmem_page_address(src[i].page);
713 copy_page(dst_data, src_data);
714 }
715}
716
717static struct bitmap_block_status *sector_to_bitmap_block(struct dm_integrity_c *ic, sector_t sector)
718{
719 unsigned bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
720 unsigned bitmap_block = bit / (BITMAP_BLOCK_SIZE * 8);
721
722 BUG_ON(bitmap_block >= ic->n_bitmap_blocks);
723 return &ic->bbs[bitmap_block];
724}
725
726static void access_journal_check(struct dm_integrity_c *ic, unsigned section, unsigned offset,
727 bool e, const char *function)
728{
729#if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY)
730 unsigned limit = e ? ic->journal_section_entries : ic->journal_section_sectors;
731
732 if (unlikely(section >= ic->journal_sections) ||
733 unlikely(offset >= limit)) {
734 DMCRIT("%s: invalid access at (%u,%u), limit (%u,%u)",
735 function, section, offset, ic->journal_sections, limit);
736 BUG();
737 }
738#endif
739}
740
741static void page_list_location(struct dm_integrity_c *ic, unsigned section, unsigned offset,
742 unsigned *pl_index, unsigned *pl_offset)
743{
744 unsigned sector;
745
746 access_journal_check(ic, section, offset, false, "page_list_location");
747
748 sector = section * ic->journal_section_sectors + offset;
749
750 *pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
751 *pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
752}
753
754static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl,
755 unsigned section, unsigned offset, unsigned *n_sectors)
756{
757 unsigned pl_index, pl_offset;
758 char *va;
759
760 page_list_location(ic, section, offset, &pl_index, &pl_offset);
761
762 if (n_sectors)
763 *n_sectors = (PAGE_SIZE - pl_offset) >> SECTOR_SHIFT;
764
765 va = lowmem_page_address(pl[pl_index].page);
766
767 return (struct journal_sector *)(va + pl_offset);
768}
769
770static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset)
771{
772 return access_page_list(ic, ic->journal, section, offset, NULL);
773}
774
775static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned section, unsigned n)
776{
777 unsigned rel_sector, offset;
778 struct journal_sector *js;
779
780 access_journal_check(ic, section, n, true, "access_journal_entry");
781
782 rel_sector = n % JOURNAL_BLOCK_SECTORS;
783 offset = n / JOURNAL_BLOCK_SECTORS;
784
785 js = access_journal(ic, section, rel_sector);
786 return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size);
787}
788
789static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned section, unsigned n)
790{
791 n <<= ic->sb->log2_sectors_per_block;
792
793 n += JOURNAL_BLOCK_SECTORS;
794
795 access_journal_check(ic, section, n, false, "access_journal_data");
796
797 return access_journal(ic, section, n);
798}
799
800static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result[JOURNAL_MAC_SIZE])
801{
802 SHASH_DESC_ON_STACK(desc, ic->journal_mac);
803 int r;
804 unsigned j, size;
805
806 desc->tfm = ic->journal_mac;
807
808 r = crypto_shash_init(desc);
809 if (unlikely(r < 0)) {
810 dm_integrity_io_error(ic, "crypto_shash_init", r);
811 goto err;
812 }
813
814 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
815 __le64 section_le;
816
817 r = crypto_shash_update(desc, (__u8 *)&ic->sb->salt, SALT_SIZE);
818 if (unlikely(r < 0)) {
819 dm_integrity_io_error(ic, "crypto_shash_update", r);
820 goto err;
821 }
822
823 section_le = cpu_to_le64(section);
824 r = crypto_shash_update(desc, (__u8 *)§ion_le, sizeof section_le);
825 if (unlikely(r < 0)) {
826 dm_integrity_io_error(ic, "crypto_shash_update", r);
827 goto err;
828 }
829 }
830
831 for (j = 0; j < ic->journal_section_entries; j++) {
832 struct journal_entry *je = access_journal_entry(ic, section, j);
833 r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof je->u.sector);
834 if (unlikely(r < 0)) {
835 dm_integrity_io_error(ic, "crypto_shash_update", r);
836 goto err;
837 }
838 }
839
840 size = crypto_shash_digestsize(ic->journal_mac);
841
842 if (likely(size <= JOURNAL_MAC_SIZE)) {
843 r = crypto_shash_final(desc, result);
844 if (unlikely(r < 0)) {
845 dm_integrity_io_error(ic, "crypto_shash_final", r);
846 goto err;
847 }
848 memset(result + size, 0, JOURNAL_MAC_SIZE - size);
849 } else {
850 __u8 digest[HASH_MAX_DIGESTSIZE];
851
852 if (WARN_ON(size > sizeof(digest))) {
853 dm_integrity_io_error(ic, "digest_size", -EINVAL);
854 goto err;
855 }
856 r = crypto_shash_final(desc, digest);
857 if (unlikely(r < 0)) {
858 dm_integrity_io_error(ic, "crypto_shash_final", r);
859 goto err;
860 }
861 memcpy(result, digest, JOURNAL_MAC_SIZE);
862 }
863
864 return;
865err:
866 memset(result, 0, JOURNAL_MAC_SIZE);
867}
868
869static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr)
870{
871 __u8 result[JOURNAL_MAC_SIZE];
872 unsigned j;
873
874 if (!ic->journal_mac)
875 return;
876
877 section_mac(ic, section, result);
878
879 for (j = 0; j < JOURNAL_BLOCK_SECTORS; j++) {
880 struct journal_sector *js = access_journal(ic, section, j);
881
882 if (likely(wr))
883 memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR);
884 else {
885 if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR)) {
886 dm_integrity_io_error(ic, "journal mac", -EILSEQ);
887 dm_audit_log_target(DM_MSG_PREFIX, "mac-journal", ic->ti, 0);
888 }
889 }
890 }
891}
892
893static void complete_journal_op(void *context)
894{
895 struct journal_completion *comp = context;
896 BUG_ON(!atomic_read(&comp->in_flight));
897 if (likely(atomic_dec_and_test(&comp->in_flight)))
898 complete(&comp->comp);
899}
900
901static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
902 unsigned n_sections, struct journal_completion *comp)
903{
904 struct async_submit_ctl submit;
905 size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT;
906 unsigned pl_index, pl_offset, section_index;
907 struct page_list *source_pl, *target_pl;
908
909 if (likely(encrypt)) {
910 source_pl = ic->journal;
911 target_pl = ic->journal_io;
912 } else {
913 source_pl = ic->journal_io;
914 target_pl = ic->journal;
915 }
916
917 page_list_location(ic, section, 0, &pl_index, &pl_offset);
918
919 atomic_add(roundup(pl_offset + n_bytes, PAGE_SIZE) >> PAGE_SHIFT, &comp->in_flight);
920
921 init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, complete_journal_op, comp, NULL);
922
923 section_index = pl_index;
924
925 do {
926 size_t this_step;
927 struct page *src_pages[2];
928 struct page *dst_page;
929
930 while (unlikely(pl_index == section_index)) {
931 unsigned dummy;
932 if (likely(encrypt))
933 rw_section_mac(ic, section, true);
934 section++;
935 n_sections--;
936 if (!n_sections)
937 break;
938 page_list_location(ic, section, 0, §ion_index, &dummy);
939 }
940
941 this_step = min(n_bytes, (size_t)PAGE_SIZE - pl_offset);
942 dst_page = target_pl[pl_index].page;
943 src_pages[0] = source_pl[pl_index].page;
944 src_pages[1] = ic->journal_xor[pl_index].page;
945
946 async_xor(dst_page, src_pages, pl_offset, 2, this_step, &submit);
947
948 pl_index++;
949 pl_offset = 0;
950 n_bytes -= this_step;
951 } while (n_bytes);
952
953 BUG_ON(n_sections);
954
955 async_tx_issue_pending_all();
956}
957
958static void complete_journal_encrypt(struct crypto_async_request *req, int err)
959{
960 struct journal_completion *comp = req->data;
961 if (unlikely(err)) {
962 if (likely(err == -EINPROGRESS)) {
963 complete(&comp->ic->crypto_backoff);
964 return;
965 }
966 dm_integrity_io_error(comp->ic, "asynchronous encrypt", err);
967 }
968 complete_journal_op(comp);
969}
970
971static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp)
972{
973 int r;
974 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
975 complete_journal_encrypt, comp);
976 if (likely(encrypt))
977 r = crypto_skcipher_encrypt(req);
978 else
979 r = crypto_skcipher_decrypt(req);
980 if (likely(!r))
981 return false;
982 if (likely(r == -EINPROGRESS))
983 return true;
984 if (likely(r == -EBUSY)) {
985 wait_for_completion(&comp->ic->crypto_backoff);
986 reinit_completion(&comp->ic->crypto_backoff);
987 return true;
988 }
989 dm_integrity_io_error(comp->ic, "encrypt", r);
990 return false;
991}
992
993static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
994 unsigned n_sections, struct journal_completion *comp)
995{
996 struct scatterlist **source_sg;
997 struct scatterlist **target_sg;
998
999 atomic_add(2, &comp->in_flight);
1000
1001 if (likely(encrypt)) {
1002 source_sg = ic->journal_scatterlist;
1003 target_sg = ic->journal_io_scatterlist;
1004 } else {
1005 source_sg = ic->journal_io_scatterlist;
1006 target_sg = ic->journal_scatterlist;
1007 }
1008
1009 do {
1010 struct skcipher_request *req;
1011 unsigned ivsize;
1012 char *iv;
1013
1014 if (likely(encrypt))
1015 rw_section_mac(ic, section, true);
1016
1017 req = ic->sk_requests[section];
1018 ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
1019 iv = req->iv;
1020
1021 memcpy(iv, iv + ivsize, ivsize);
1022
1023 req->src = source_sg[section];
1024 req->dst = target_sg[section];
1025
1026 if (unlikely(do_crypt(encrypt, req, comp)))
1027 atomic_inc(&comp->in_flight);
1028
1029 section++;
1030 n_sections--;
1031 } while (n_sections);
1032
1033 atomic_dec(&comp->in_flight);
1034 complete_journal_op(comp);
1035}
1036
1037static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
1038 unsigned n_sections, struct journal_completion *comp)
1039{
1040 if (ic->journal_xor)
1041 return xor_journal(ic, encrypt, section, n_sections, comp);
1042 else
1043 return crypt_journal(ic, encrypt, section, n_sections, comp);
1044}
1045
1046static void complete_journal_io(unsigned long error, void *context)
1047{
1048 struct journal_completion *comp = context;
1049 if (unlikely(error != 0))
1050 dm_integrity_io_error(comp->ic, "writing journal", -EIO);
1051 complete_journal_op(comp);
1052}
1053
1054static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf,
1055 unsigned sector, unsigned n_sectors,
1056 struct journal_completion *comp)
1057{
1058 struct dm_io_request io_req;
1059 struct dm_io_region io_loc;
1060 unsigned pl_index, pl_offset;
1061 int r;
1062
1063 if (unlikely(dm_integrity_failed(ic))) {
1064 if (comp)
1065 complete_journal_io(-1UL, comp);
1066 return;
1067 }
1068
1069 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
1070 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
1071
1072 io_req.bi_opf = opf;
1073 io_req.mem.type = DM_IO_PAGE_LIST;
1074 if (ic->journal_io)
1075 io_req.mem.ptr.pl = &ic->journal_io[pl_index];
1076 else
1077 io_req.mem.ptr.pl = &ic->journal[pl_index];
1078 io_req.mem.offset = pl_offset;
1079 if (likely(comp != NULL)) {
1080 io_req.notify.fn = complete_journal_io;
1081 io_req.notify.context = comp;
1082 } else {
1083 io_req.notify.fn = NULL;
1084 }
1085 io_req.client = ic->io;
1086 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
1087 io_loc.sector = ic->start + SB_SECTORS + sector;
1088 io_loc.count = n_sectors;
1089
1090 r = dm_io(&io_req, 1, &io_loc, NULL);
1091 if (unlikely(r)) {
1092 dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ?
1093 "reading journal" : "writing journal", r);
1094 if (comp) {
1095 WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
1096 complete_journal_io(-1UL, comp);
1097 }
1098 }
1099}
1100
1101static void rw_journal(struct dm_integrity_c *ic, blk_opf_t opf,
1102 unsigned section, unsigned n_sections,
1103 struct journal_completion *comp)
1104{
1105 unsigned sector, n_sectors;
1106
1107 sector = section * ic->journal_section_sectors;
1108 n_sectors = n_sections * ic->journal_section_sectors;
1109
1110 rw_journal_sectors(ic, opf, sector, n_sectors, comp);
1111}
1112
1113static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsigned commit_sections)
1114{
1115 struct journal_completion io_comp;
1116 struct journal_completion crypt_comp_1;
1117 struct journal_completion crypt_comp_2;
1118 unsigned i;
1119
1120 io_comp.ic = ic;
1121 init_completion(&io_comp.comp);
1122
1123 if (commit_start + commit_sections <= ic->journal_sections) {
1124 io_comp.in_flight = (atomic_t)ATOMIC_INIT(1);
1125 if (ic->journal_io) {
1126 crypt_comp_1.ic = ic;
1127 init_completion(&crypt_comp_1.comp);
1128 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1129 encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1);
1130 wait_for_completion_io(&crypt_comp_1.comp);
1131 } else {
1132 for (i = 0; i < commit_sections; i++)
1133 rw_section_mac(ic, commit_start + i, true);
1134 }
1135 rw_journal(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, commit_start,
1136 commit_sections, &io_comp);
1137 } else {
1138 unsigned to_end;
1139 io_comp.in_flight = (atomic_t)ATOMIC_INIT(2);
1140 to_end = ic->journal_sections - commit_start;
1141 if (ic->journal_io) {
1142 crypt_comp_1.ic = ic;
1143 init_completion(&crypt_comp_1.comp);
1144 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1145 encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
1146 if (try_wait_for_completion(&crypt_comp_1.comp)) {
1147 rw_journal(ic, REQ_OP_WRITE | REQ_FUA,
1148 commit_start, to_end, &io_comp);
1149 reinit_completion(&crypt_comp_1.comp);
1150 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1151 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
1152 wait_for_completion_io(&crypt_comp_1.comp);
1153 } else {
1154 crypt_comp_2.ic = ic;
1155 init_completion(&crypt_comp_2.comp);
1156 crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
1157 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
1158 wait_for_completion_io(&crypt_comp_1.comp);
1159 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp);
1160 wait_for_completion_io(&crypt_comp_2.comp);
1161 }
1162 } else {
1163 for (i = 0; i < to_end; i++)
1164 rw_section_mac(ic, commit_start + i, true);
1165 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp);
1166 for (i = 0; i < commit_sections - to_end; i++)
1167 rw_section_mac(ic, i, true);
1168 }
1169 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, 0, commit_sections - to_end, &io_comp);
1170 }
1171
1172 wait_for_completion_io(&io_comp.comp);
1173}
1174
1175static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset,
1176 unsigned n_sectors, sector_t target, io_notify_fn fn, void *data)
1177{
1178 struct dm_io_request io_req;
1179 struct dm_io_region io_loc;
1180 int r;
1181 unsigned sector, pl_index, pl_offset;
1182
1183 BUG_ON((target | n_sectors | offset) & (unsigned)(ic->sectors_per_block - 1));
1184
1185 if (unlikely(dm_integrity_failed(ic))) {
1186 fn(-1UL, data);
1187 return;
1188 }
1189
1190 sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset;
1191
1192 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
1193 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
1194
1195 io_req.bi_opf = REQ_OP_WRITE;
1196 io_req.mem.type = DM_IO_PAGE_LIST;
1197 io_req.mem.ptr.pl = &ic->journal[pl_index];
1198 io_req.mem.offset = pl_offset;
1199 io_req.notify.fn = fn;
1200 io_req.notify.context = data;
1201 io_req.client = ic->io;
1202 io_loc.bdev = ic->dev->bdev;
1203 io_loc.sector = target;
1204 io_loc.count = n_sectors;
1205
1206 r = dm_io(&io_req, 1, &io_loc, NULL);
1207 if (unlikely(r)) {
1208 WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
1209 fn(-1UL, data);
1210 }
1211}
1212
1213static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2)
1214{
1215 return range1->logical_sector < range2->logical_sector + range2->n_sectors &&
1216 range1->logical_sector + range1->n_sectors > range2->logical_sector;
1217}
1218
1219static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting)
1220{
1221 struct rb_node **n = &ic->in_progress.rb_node;
1222 struct rb_node *parent;
1223
1224 BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned)(ic->sectors_per_block - 1));
1225
1226 if (likely(check_waiting)) {
1227 struct dm_integrity_range *range;
1228 list_for_each_entry(range, &ic->wait_list, wait_entry) {
1229 if (unlikely(ranges_overlap(range, new_range)))
1230 return false;
1231 }
1232 }
1233
1234 parent = NULL;
1235
1236 while (*n) {
1237 struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node);
1238
1239 parent = *n;
1240 if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector) {
1241 n = &range->node.rb_left;
1242 } else if (new_range->logical_sector >= range->logical_sector + range->n_sectors) {
1243 n = &range->node.rb_right;
1244 } else {
1245 return false;
1246 }
1247 }
1248
1249 rb_link_node(&new_range->node, parent, n);
1250 rb_insert_color(&new_range->node, &ic->in_progress);
1251
1252 return true;
1253}
1254
1255static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range)
1256{
1257 rb_erase(&range->node, &ic->in_progress);
1258 while (unlikely(!list_empty(&ic->wait_list))) {
1259 struct dm_integrity_range *last_range =
1260 list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry);
1261 struct task_struct *last_range_task;
1262 last_range_task = last_range->task;
1263 list_del(&last_range->wait_entry);
1264 if (!add_new_range(ic, last_range, false)) {
1265 last_range->task = last_range_task;
1266 list_add(&last_range->wait_entry, &ic->wait_list);
1267 break;
1268 }
1269 last_range->waiting = false;
1270 wake_up_process(last_range_task);
1271 }
1272}
1273
1274static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range)
1275{
1276 unsigned long flags;
1277
1278 spin_lock_irqsave(&ic->endio_wait.lock, flags);
1279 remove_range_unlocked(ic, range);
1280 spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1281}
1282
1283static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
1284{
1285 new_range->waiting = true;
1286 list_add_tail(&new_range->wait_entry, &ic->wait_list);
1287 new_range->task = current;
1288 do {
1289 __set_current_state(TASK_UNINTERRUPTIBLE);
1290 spin_unlock_irq(&ic->endio_wait.lock);
1291 io_schedule();
1292 spin_lock_irq(&ic->endio_wait.lock);
1293 } while (unlikely(new_range->waiting));
1294}
1295
1296static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
1297{
1298 if (unlikely(!add_new_range(ic, new_range, true)))
1299 wait_and_add_new_range(ic, new_range);
1300}
1301
1302static void init_journal_node(struct journal_node *node)
1303{
1304 RB_CLEAR_NODE(&node->node);
1305 node->sector = (sector_t)-1;
1306}
1307
1308static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector)
1309{
1310 struct rb_node **link;
1311 struct rb_node *parent;
1312
1313 node->sector = sector;
1314 BUG_ON(!RB_EMPTY_NODE(&node->node));
1315
1316 link = &ic->journal_tree_root.rb_node;
1317 parent = NULL;
1318
1319 while (*link) {
1320 struct journal_node *j;
1321 parent = *link;
1322 j = container_of(parent, struct journal_node, node);
1323 if (sector < j->sector)
1324 link = &j->node.rb_left;
1325 else
1326 link = &j->node.rb_right;
1327 }
1328
1329 rb_link_node(&node->node, parent, link);
1330 rb_insert_color(&node->node, &ic->journal_tree_root);
1331}
1332
1333static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node)
1334{
1335 BUG_ON(RB_EMPTY_NODE(&node->node));
1336 rb_erase(&node->node, &ic->journal_tree_root);
1337 init_journal_node(node);
1338}
1339
1340#define NOT_FOUND (-1U)
1341
1342static unsigned find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector)
1343{
1344 struct rb_node *n = ic->journal_tree_root.rb_node;
1345 unsigned found = NOT_FOUND;
1346 *next_sector = (sector_t)-1;
1347 while (n) {
1348 struct journal_node *j = container_of(n, struct journal_node, node);
1349 if (sector == j->sector) {
1350 found = j - ic->journal_tree;
1351 }
1352 if (sector < j->sector) {
1353 *next_sector = j->sector;
1354 n = j->node.rb_left;
1355 } else {
1356 n = j->node.rb_right;
1357 }
1358 }
1359
1360 return found;
1361}
1362
1363static bool test_journal_node(struct dm_integrity_c *ic, unsigned pos, sector_t sector)
1364{
1365 struct journal_node *node, *next_node;
1366 struct rb_node *next;
1367
1368 if (unlikely(pos >= ic->journal_entries))
1369 return false;
1370 node = &ic->journal_tree[pos];
1371 if (unlikely(RB_EMPTY_NODE(&node->node)))
1372 return false;
1373 if (unlikely(node->sector != sector))
1374 return false;
1375
1376 next = rb_next(&node->node);
1377 if (unlikely(!next))
1378 return true;
1379
1380 next_node = container_of(next, struct journal_node, node);
1381 return next_node->sector != sector;
1382}
1383
1384static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node)
1385{
1386 struct rb_node *next;
1387 struct journal_node *next_node;
1388 unsigned next_section;
1389
1390 BUG_ON(RB_EMPTY_NODE(&node->node));
1391
1392 next = rb_next(&node->node);
1393 if (unlikely(!next))
1394 return false;
1395
1396 next_node = container_of(next, struct journal_node, node);
1397
1398 if (next_node->sector != node->sector)
1399 return false;
1400
1401 next_section = (unsigned)(next_node - ic->journal_tree) / ic->journal_section_entries;
1402 if (next_section >= ic->committed_section &&
1403 next_section < ic->committed_section + ic->n_committed_sections)
1404 return true;
1405 if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections)
1406 return true;
1407
1408 return false;
1409}
1410
1411#define TAG_READ 0
1412#define TAG_WRITE 1
1413#define TAG_CMP 2
1414
1415static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block,
1416 unsigned *metadata_offset, unsigned total_size, int op)
1417{
1418#define MAY_BE_FILLER 1
1419#define MAY_BE_HASH 2
1420 unsigned hash_offset = 0;
1421 unsigned may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0);
1422
1423 do {
1424 unsigned char *data, *dp;
1425 struct dm_buffer *b;
1426 unsigned to_copy;
1427 int r;
1428
1429 r = dm_integrity_failed(ic);
1430 if (unlikely(r))
1431 return r;
1432
1433 data = dm_bufio_read(ic->bufio, *metadata_block, &b);
1434 if (IS_ERR(data))
1435 return PTR_ERR(data);
1436
1437 to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size);
1438 dp = data + *metadata_offset;
1439 if (op == TAG_READ) {
1440 memcpy(tag, dp, to_copy);
1441 } else if (op == TAG_WRITE) {
1442 if (memcmp(dp, tag, to_copy)) {
1443 memcpy(dp, tag, to_copy);
1444 dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy);
1445 }
1446 } else {
1447 /* e.g.: op == TAG_CMP */
1448
1449 if (likely(is_power_of_2(ic->tag_size))) {
1450 if (unlikely(memcmp(dp, tag, to_copy)))
1451 if (unlikely(!ic->discard) ||
1452 unlikely(memchr_inv(dp, DISCARD_FILLER, to_copy) != NULL)) {
1453 goto thorough_test;
1454 }
1455 } else {
1456 unsigned i, ts;
1457thorough_test:
1458 ts = total_size;
1459
1460 for (i = 0; i < to_copy; i++, ts--) {
1461 if (unlikely(dp[i] != tag[i]))
1462 may_be &= ~MAY_BE_HASH;
1463 if (likely(dp[i] != DISCARD_FILLER))
1464 may_be &= ~MAY_BE_FILLER;
1465 hash_offset++;
1466 if (unlikely(hash_offset == ic->tag_size)) {
1467 if (unlikely(!may_be)) {
1468 dm_bufio_release(b);
1469 return ts;
1470 }
1471 hash_offset = 0;
1472 may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0);
1473 }
1474 }
1475 }
1476 }
1477 dm_bufio_release(b);
1478
1479 tag += to_copy;
1480 *metadata_offset += to_copy;
1481 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) {
1482 (*metadata_block)++;
1483 *metadata_offset = 0;
1484 }
1485
1486 if (unlikely(!is_power_of_2(ic->tag_size))) {
1487 hash_offset = (hash_offset + to_copy) % ic->tag_size;
1488 }
1489
1490 total_size -= to_copy;
1491 } while (unlikely(total_size));
1492
1493 return 0;
1494#undef MAY_BE_FILLER
1495#undef MAY_BE_HASH
1496}
1497
1498struct flush_request {
1499 struct dm_io_request io_req;
1500 struct dm_io_region io_reg;
1501 struct dm_integrity_c *ic;
1502 struct completion comp;
1503};
1504
1505static void flush_notify(unsigned long error, void *fr_)
1506{
1507 struct flush_request *fr = fr_;
1508 if (unlikely(error != 0))
1509 dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO);
1510 complete(&fr->comp);
1511}
1512
1513static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data)
1514{
1515 int r;
1516
1517 struct flush_request fr;
1518
1519 if (!ic->meta_dev)
1520 flush_data = false;
1521 if (flush_data) {
1522 fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC,
1523 fr.io_req.mem.type = DM_IO_KMEM,
1524 fr.io_req.mem.ptr.addr = NULL,
1525 fr.io_req.notify.fn = flush_notify,
1526 fr.io_req.notify.context = &fr;
1527 fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio),
1528 fr.io_reg.bdev = ic->dev->bdev,
1529 fr.io_reg.sector = 0,
1530 fr.io_reg.count = 0,
1531 fr.ic = ic;
1532 init_completion(&fr.comp);
1533 r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL);
1534 BUG_ON(r);
1535 }
1536
1537 r = dm_bufio_write_dirty_buffers(ic->bufio);
1538 if (unlikely(r))
1539 dm_integrity_io_error(ic, "writing tags", r);
1540
1541 if (flush_data)
1542 wait_for_completion(&fr.comp);
1543}
1544
1545static void sleep_on_endio_wait(struct dm_integrity_c *ic)
1546{
1547 DECLARE_WAITQUEUE(wait, current);
1548 __add_wait_queue(&ic->endio_wait, &wait);
1549 __set_current_state(TASK_UNINTERRUPTIBLE);
1550 spin_unlock_irq(&ic->endio_wait.lock);
1551 io_schedule();
1552 spin_lock_irq(&ic->endio_wait.lock);
1553 __remove_wait_queue(&ic->endio_wait, &wait);
1554}
1555
1556static void autocommit_fn(struct timer_list *t)
1557{
1558 struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer);
1559
1560 if (likely(!dm_integrity_failed(ic)))
1561 queue_work(ic->commit_wq, &ic->commit_work);
1562}
1563
1564static void schedule_autocommit(struct dm_integrity_c *ic)
1565{
1566 if (!timer_pending(&ic->autocommit_timer))
1567 mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies);
1568}
1569
1570static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1571{
1572 struct bio *bio;
1573 unsigned long flags;
1574
1575 spin_lock_irqsave(&ic->endio_wait.lock, flags);
1576 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1577 bio_list_add(&ic->flush_bio_list, bio);
1578 spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1579
1580 queue_work(ic->commit_wq, &ic->commit_work);
1581}
1582
1583static void do_endio(struct dm_integrity_c *ic, struct bio *bio)
1584{
1585 int r = dm_integrity_failed(ic);
1586 if (unlikely(r) && !bio->bi_status)
1587 bio->bi_status = errno_to_blk_status(r);
1588 if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) {
1589 unsigned long flags;
1590 spin_lock_irqsave(&ic->endio_wait.lock, flags);
1591 bio_list_add(&ic->synchronous_bios, bio);
1592 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
1593 spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1594 return;
1595 }
1596 bio_endio(bio);
1597}
1598
1599static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1600{
1601 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1602
1603 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))
1604 submit_flush_bio(ic, dio);
1605 else
1606 do_endio(ic, bio);
1607}
1608
1609static void dec_in_flight(struct dm_integrity_io *dio)
1610{
1611 if (atomic_dec_and_test(&dio->in_flight)) {
1612 struct dm_integrity_c *ic = dio->ic;
1613 struct bio *bio;
1614
1615 remove_range(ic, &dio->range);
1616
1617 if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))
1618 schedule_autocommit(ic);
1619
1620 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1621
1622 if (unlikely(dio->bi_status) && !bio->bi_status)
1623 bio->bi_status = dio->bi_status;
1624 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {
1625 dio->range.logical_sector += dio->range.n_sectors;
1626 bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);
1627 INIT_WORK(&dio->work, integrity_bio_wait);
1628 queue_work(ic->offload_wq, &dio->work);
1629 return;
1630 }
1631 do_endio_flush(ic, dio);
1632 }
1633}
1634
1635static void integrity_end_io(struct bio *bio)
1636{
1637 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1638
1639 dm_bio_restore(&dio->bio_details, bio);
1640 if (bio->bi_integrity)
1641 bio->bi_opf |= REQ_INTEGRITY;
1642
1643 if (dio->completion)
1644 complete(dio->completion);
1645
1646 dec_in_flight(dio);
1647}
1648
1649static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector,
1650 const char *data, char *result)
1651{
1652 __le64 sector_le = cpu_to_le64(sector);
1653 SHASH_DESC_ON_STACK(req, ic->internal_hash);
1654 int r;
1655 unsigned digest_size;
1656
1657 req->tfm = ic->internal_hash;
1658
1659 r = crypto_shash_init(req);
1660 if (unlikely(r < 0)) {
1661 dm_integrity_io_error(ic, "crypto_shash_init", r);
1662 goto failed;
1663 }
1664
1665 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
1666 r = crypto_shash_update(req, (__u8 *)&ic->sb->salt, SALT_SIZE);
1667 if (unlikely(r < 0)) {
1668 dm_integrity_io_error(ic, "crypto_shash_update", r);
1669 goto failed;
1670 }
1671 }
1672
1673 r = crypto_shash_update(req, (const __u8 *)§or_le, sizeof sector_le);
1674 if (unlikely(r < 0)) {
1675 dm_integrity_io_error(ic, "crypto_shash_update", r);
1676 goto failed;
1677 }
1678
1679 r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT);
1680 if (unlikely(r < 0)) {
1681 dm_integrity_io_error(ic, "crypto_shash_update", r);
1682 goto failed;
1683 }
1684
1685 r = crypto_shash_final(req, result);
1686 if (unlikely(r < 0)) {
1687 dm_integrity_io_error(ic, "crypto_shash_final", r);
1688 goto failed;
1689 }
1690
1691 digest_size = crypto_shash_digestsize(ic->internal_hash);
1692 if (unlikely(digest_size < ic->tag_size))
1693 memset(result + digest_size, 0, ic->tag_size - digest_size);
1694
1695 return;
1696
1697failed:
1698 /* this shouldn't happen anyway, the hash functions have no reason to fail */
1699 get_random_bytes(result, ic->tag_size);
1700}
1701
1702static void integrity_metadata(struct work_struct *w)
1703{
1704 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1705 struct dm_integrity_c *ic = dio->ic;
1706
1707 int r;
1708
1709 if (ic->internal_hash) {
1710 struct bvec_iter iter;
1711 struct bio_vec bv;
1712 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1713 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1714 char *checksums;
1715 unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
1716 char checksums_onstack[max((size_t)HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
1717 sector_t sector;
1718 unsigned sectors_to_process;
1719
1720 if (unlikely(ic->mode == 'R'))
1721 goto skip_io;
1722
1723 if (likely(dio->op != REQ_OP_DISCARD))
1724 checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
1725 GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1726 else
1727 checksums = kmalloc(PAGE_SIZE, GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1728 if (!checksums) {
1729 checksums = checksums_onstack;
1730 if (WARN_ON(extra_space &&
1731 digest_size > sizeof(checksums_onstack))) {
1732 r = -EINVAL;
1733 goto error;
1734 }
1735 }
1736
1737 if (unlikely(dio->op == REQ_OP_DISCARD)) {
1738 sector_t bi_sector = dio->bio_details.bi_iter.bi_sector;
1739 unsigned bi_size = dio->bio_details.bi_iter.bi_size;
1740 unsigned max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE;
1741 unsigned max_blocks = max_size / ic->tag_size;
1742 memset(checksums, DISCARD_FILLER, max_size);
1743
1744 while (bi_size) {
1745 unsigned this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block);
1746 this_step_blocks = min(this_step_blocks, max_blocks);
1747 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1748 this_step_blocks * ic->tag_size, TAG_WRITE);
1749 if (unlikely(r)) {
1750 if (likely(checksums != checksums_onstack))
1751 kfree(checksums);
1752 goto error;
1753 }
1754
1755 /*if (bi_size < this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block)) {
1756 printk("BUGG: bi_sector: %llx, bi_size: %u\n", bi_sector, bi_size);
1757 printk("BUGG: this_step_blocks: %u\n", this_step_blocks);
1758 BUG();
1759 }*/
1760 bi_size -= this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block);
1761 bi_sector += this_step_blocks << ic->sb->log2_sectors_per_block;
1762 }
1763
1764 if (likely(checksums != checksums_onstack))
1765 kfree(checksums);
1766 goto skip_io;
1767 }
1768
1769 sector = dio->range.logical_sector;
1770 sectors_to_process = dio->range.n_sectors;
1771
1772 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1773 unsigned pos;
1774 char *mem, *checksums_ptr;
1775
1776again:
1777 mem = bvec_kmap_local(&bv);
1778 pos = 0;
1779 checksums_ptr = checksums;
1780 do {
1781 integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr);
1782 checksums_ptr += ic->tag_size;
1783 sectors_to_process -= ic->sectors_per_block;
1784 pos += ic->sectors_per_block << SECTOR_SHIFT;
1785 sector += ic->sectors_per_block;
1786 } while (pos < bv.bv_len && sectors_to_process && checksums != checksums_onstack);
1787 kunmap_local(mem);
1788
1789 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1790 checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);
1791 if (unlikely(r)) {
1792 if (r > 0) {
1793 sector_t s;
1794
1795 s = sector - ((r + ic->tag_size - 1) / ic->tag_size);
1796 DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
1797 bio->bi_bdev, s);
1798 r = -EILSEQ;
1799 atomic64_inc(&ic->number_of_mismatches);
1800 dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum",
1801 bio, s, 0);
1802 }
1803 if (likely(checksums != checksums_onstack))
1804 kfree(checksums);
1805 goto error;
1806 }
1807
1808 if (!sectors_to_process)
1809 break;
1810
1811 if (unlikely(pos < bv.bv_len)) {
1812 bv.bv_offset += pos;
1813 bv.bv_len -= pos;
1814 goto again;
1815 }
1816 }
1817
1818 if (likely(checksums != checksums_onstack))
1819 kfree(checksums);
1820 } else {
1821 struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;
1822
1823 if (bip) {
1824 struct bio_vec biv;
1825 struct bvec_iter iter;
1826 unsigned data_to_process = dio->range.n_sectors;
1827 sector_to_block(ic, data_to_process);
1828 data_to_process *= ic->tag_size;
1829
1830 bip_for_each_vec(biv, bip, iter) {
1831 unsigned char *tag;
1832 unsigned this_len;
1833
1834 BUG_ON(PageHighMem(biv.bv_page));
1835 tag = bvec_virt(&biv);
1836 this_len = min(biv.bv_len, data_to_process);
1837 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,
1838 this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);
1839 if (unlikely(r))
1840 goto error;
1841 data_to_process -= this_len;
1842 if (!data_to_process)
1843 break;
1844 }
1845 }
1846 }
1847skip_io:
1848 dec_in_flight(dio);
1849 return;
1850error:
1851 dio->bi_status = errno_to_blk_status(r);
1852 dec_in_flight(dio);
1853}
1854
1855static int dm_integrity_map(struct dm_target *ti, struct bio *bio)
1856{
1857 struct dm_integrity_c *ic = ti->private;
1858 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1859 struct bio_integrity_payload *bip;
1860
1861 sector_t area, offset;
1862
1863 dio->ic = ic;
1864 dio->bi_status = 0;
1865 dio->op = bio_op(bio);
1866
1867 if (unlikely(dio->op == REQ_OP_DISCARD)) {
1868 if (ti->max_io_len) {
1869 sector_t sec = dm_target_offset(ti, bio->bi_iter.bi_sector);
1870 unsigned log2_max_io_len = __fls(ti->max_io_len);
1871 sector_t start_boundary = sec >> log2_max_io_len;
1872 sector_t end_boundary = (sec + bio_sectors(bio) - 1) >> log2_max_io_len;
1873 if (start_boundary < end_boundary) {
1874 sector_t len = ti->max_io_len - (sec & (ti->max_io_len - 1));
1875 dm_accept_partial_bio(bio, len);
1876 }
1877 }
1878 }
1879
1880 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1881 submit_flush_bio(ic, dio);
1882 return DM_MAPIO_SUBMITTED;
1883 }
1884
1885 dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1886 dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;
1887 if (unlikely(dio->fua)) {
1888 /*
1889 * Don't pass down the FUA flag because we have to flush
1890 * disk cache anyway.
1891 */
1892 bio->bi_opf &= ~REQ_FUA;
1893 }
1894 if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {
1895 DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx",
1896 dio->range.logical_sector, bio_sectors(bio),
1897 ic->provided_data_sectors);
1898 return DM_MAPIO_KILL;
1899 }
1900 if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1))) {
1901 DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x",
1902 ic->sectors_per_block,
1903 dio->range.logical_sector, bio_sectors(bio));
1904 return DM_MAPIO_KILL;
1905 }
1906
1907 if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) {
1908 struct bvec_iter iter;
1909 struct bio_vec bv;
1910 bio_for_each_segment(bv, bio, iter) {
1911 if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) {
1912 DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary",
1913 bv.bv_offset, bv.bv_len, ic->sectors_per_block);
1914 return DM_MAPIO_KILL;
1915 }
1916 }
1917 }
1918
1919 bip = bio_integrity(bio);
1920 if (!ic->internal_hash) {
1921 if (bip) {
1922 unsigned wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block;
1923 if (ic->log2_tag_size >= 0)
1924 wanted_tag_size <<= ic->log2_tag_size;
1925 else
1926 wanted_tag_size *= ic->tag_size;
1927 if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) {
1928 DMERR("Invalid integrity data size %u, expected %u",
1929 bip->bip_iter.bi_size, wanted_tag_size);
1930 return DM_MAPIO_KILL;
1931 }
1932 }
1933 } else {
1934 if (unlikely(bip != NULL)) {
1935 DMERR("Unexpected integrity data when using internal hash");
1936 return DM_MAPIO_KILL;
1937 }
1938 }
1939
1940 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))
1941 return DM_MAPIO_KILL;
1942
1943 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1944 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1945 bio->bi_iter.bi_sector = get_data_sector(ic, area, offset);
1946
1947 dm_integrity_map_continue(dio, true);
1948 return DM_MAPIO_SUBMITTED;
1949}
1950
1951static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,
1952 unsigned journal_section, unsigned journal_entry)
1953{
1954 struct dm_integrity_c *ic = dio->ic;
1955 sector_t logical_sector;
1956 unsigned n_sectors;
1957
1958 logical_sector = dio->range.logical_sector;
1959 n_sectors = dio->range.n_sectors;
1960 do {
1961 struct bio_vec bv = bio_iovec(bio);
1962 char *mem;
1963
1964 if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors))
1965 bv.bv_len = n_sectors << SECTOR_SHIFT;
1966 n_sectors -= bv.bv_len >> SECTOR_SHIFT;
1967 bio_advance_iter(bio, &bio->bi_iter, bv.bv_len);
1968retry_kmap:
1969 mem = kmap_local_page(bv.bv_page);
1970 if (likely(dio->op == REQ_OP_WRITE))
1971 flush_dcache_page(bv.bv_page);
1972
1973 do {
1974 struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry);
1975
1976 if (unlikely(dio->op == REQ_OP_READ)) {
1977 struct journal_sector *js;
1978 char *mem_ptr;
1979 unsigned s;
1980
1981 if (unlikely(journal_entry_is_inprogress(je))) {
1982 flush_dcache_page(bv.bv_page);
1983 kunmap_local(mem);
1984
1985 __io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
1986 goto retry_kmap;
1987 }
1988 smp_rmb();
1989 BUG_ON(journal_entry_get_sector(je) != logical_sector);
1990 js = access_journal_data(ic, journal_section, journal_entry);
1991 mem_ptr = mem + bv.bv_offset;
1992 s = 0;
1993 do {
1994 memcpy(mem_ptr, js, JOURNAL_SECTOR_DATA);
1995 *(commit_id_t *)(mem_ptr + JOURNAL_SECTOR_DATA) = je->last_bytes[s];
1996 js++;
1997 mem_ptr += 1 << SECTOR_SHIFT;
1998 } while (++s < ic->sectors_per_block);
1999#ifdef INTERNAL_VERIFY
2000 if (ic->internal_hash) {
2001 char checksums_onstack[max((size_t)HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
2002
2003 integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack);
2004 if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) {
2005 DMERR_LIMIT("Checksum failed when reading from journal, at sector 0x%llx",
2006 logical_sector);
2007 dm_audit_log_bio(DM_MSG_PREFIX, "journal-checksum",
2008 bio, logical_sector, 0);
2009 }
2010 }
2011#endif
2012 }
2013
2014 if (!ic->internal_hash) {
2015 struct bio_integrity_payload *bip = bio_integrity(bio);
2016 unsigned tag_todo = ic->tag_size;
2017 char *tag_ptr = journal_entry_tag(ic, je);
2018
2019 if (bip) do {
2020 struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter);
2021 unsigned tag_now = min(biv.bv_len, tag_todo);
2022 char *tag_addr;
2023 BUG_ON(PageHighMem(biv.bv_page));
2024 tag_addr = bvec_virt(&biv);
2025 if (likely(dio->op == REQ_OP_WRITE))
2026 memcpy(tag_ptr, tag_addr, tag_now);
2027 else
2028 memcpy(tag_addr, tag_ptr, tag_now);
2029 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now);
2030 tag_ptr += tag_now;
2031 tag_todo -= tag_now;
2032 } while (unlikely(tag_todo)); else {
2033 if (likely(dio->op == REQ_OP_WRITE))
2034 memset(tag_ptr, 0, tag_todo);
2035 }
2036 }
2037
2038 if (likely(dio->op == REQ_OP_WRITE)) {
2039 struct journal_sector *js;
2040 unsigned s;
2041
2042 js = access_journal_data(ic, journal_section, journal_entry);
2043 memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT);
2044
2045 s = 0;
2046 do {
2047 je->last_bytes[s] = js[s].commit_id;
2048 } while (++s < ic->sectors_per_block);
2049
2050 if (ic->internal_hash) {
2051 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
2052 if (unlikely(digest_size > ic->tag_size)) {
2053 char checksums_onstack[HASH_MAX_DIGESTSIZE];
2054 integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack);
2055 memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size);
2056 } else
2057 integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je));
2058 }
2059
2060 journal_entry_set_sector(je, logical_sector);
2061 }
2062 logical_sector += ic->sectors_per_block;
2063
2064 journal_entry++;
2065 if (unlikely(journal_entry == ic->journal_section_entries)) {
2066 journal_entry = 0;
2067 journal_section++;
2068 wraparound_section(ic, &journal_section);
2069 }
2070
2071 bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT;
2072 } while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT);
2073
2074 if (unlikely(dio->op == REQ_OP_READ))
2075 flush_dcache_page(bv.bv_page);
2076 kunmap_local(mem);
2077 } while (n_sectors);
2078
2079 if (likely(dio->op == REQ_OP_WRITE)) {
2080 smp_mb();
2081 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait)))
2082 wake_up(&ic->copy_to_journal_wait);
2083 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) {
2084 queue_work(ic->commit_wq, &ic->commit_work);
2085 } else {
2086 schedule_autocommit(ic);
2087 }
2088 } else {
2089 remove_range(ic, &dio->range);
2090 }
2091
2092 if (unlikely(bio->bi_iter.bi_size)) {
2093 sector_t area, offset;
2094
2095 dio->range.logical_sector = logical_sector;
2096 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
2097 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
2098 return true;
2099 }
2100
2101 return false;
2102}
2103
2104static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)
2105{
2106 struct dm_integrity_c *ic = dio->ic;
2107 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2108 unsigned journal_section, journal_entry;
2109 unsigned journal_read_pos;
2110 struct completion read_comp;
2111 bool discard_retried = false;
2112 bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;
2113 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')
2114 need_sync_io = true;
2115
2116 if (need_sync_io && from_map) {
2117 INIT_WORK(&dio->work, integrity_bio_wait);
2118 queue_work(ic->offload_wq, &dio->work);
2119 return;
2120 }
2121
2122lock_retry:
2123 spin_lock_irq(&ic->endio_wait.lock);
2124retry:
2125 if (unlikely(dm_integrity_failed(ic))) {
2126 spin_unlock_irq(&ic->endio_wait.lock);
2127 do_endio(ic, bio);
2128 return;
2129 }
2130 dio->range.n_sectors = bio_sectors(bio);
2131 journal_read_pos = NOT_FOUND;
2132 if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {
2133 if (dio->op == REQ_OP_WRITE) {
2134 unsigned next_entry, i, pos;
2135 unsigned ws, we, range_sectors;
2136
2137 dio->range.n_sectors = min(dio->range.n_sectors,
2138 (sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block);
2139 if (unlikely(!dio->range.n_sectors)) {
2140 if (from_map)
2141 goto offload_to_thread;
2142 sleep_on_endio_wait(ic);
2143 goto retry;
2144 }
2145 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;
2146 ic->free_sectors -= range_sectors;
2147 journal_section = ic->free_section;
2148 journal_entry = ic->free_section_entry;
2149
2150 next_entry = ic->free_section_entry + range_sectors;
2151 ic->free_section_entry = next_entry % ic->journal_section_entries;
2152 ic->free_section += next_entry / ic->journal_section_entries;
2153 ic->n_uncommitted_sections += next_entry / ic->journal_section_entries;
2154 wraparound_section(ic, &ic->free_section);
2155
2156 pos = journal_section * ic->journal_section_entries + journal_entry;
2157 ws = journal_section;
2158 we = journal_entry;
2159 i = 0;
2160 do {
2161 struct journal_entry *je;
2162
2163 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);
2164 pos++;
2165 if (unlikely(pos >= ic->journal_entries))
2166 pos = 0;
2167
2168 je = access_journal_entry(ic, ws, we);
2169 BUG_ON(!journal_entry_is_unused(je));
2170 journal_entry_set_inprogress(je);
2171 we++;
2172 if (unlikely(we == ic->journal_section_entries)) {
2173 we = 0;
2174 ws++;
2175 wraparound_section(ic, &ws);
2176 }
2177 } while ((i += ic->sectors_per_block) < dio->range.n_sectors);
2178
2179 spin_unlock_irq(&ic->endio_wait.lock);
2180 goto journal_read_write;
2181 } else {
2182 sector_t next_sector;
2183 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2184 if (likely(journal_read_pos == NOT_FOUND)) {
2185 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))
2186 dio->range.n_sectors = next_sector - dio->range.logical_sector;
2187 } else {
2188 unsigned i;
2189 unsigned jp = journal_read_pos + 1;
2190 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {
2191 if (!test_journal_node(ic, jp, dio->range.logical_sector + i))
2192 break;
2193 }
2194 dio->range.n_sectors = i;
2195 }
2196 }
2197 }
2198 if (unlikely(!add_new_range(ic, &dio->range, true))) {
2199 /*
2200 * We must not sleep in the request routine because it could
2201 * stall bios on current->bio_list.
2202 * So, we offload the bio to a workqueue if we have to sleep.
2203 */
2204 if (from_map) {
2205offload_to_thread:
2206 spin_unlock_irq(&ic->endio_wait.lock);
2207 INIT_WORK(&dio->work, integrity_bio_wait);
2208 queue_work(ic->wait_wq, &dio->work);
2209 return;
2210 }
2211 if (journal_read_pos != NOT_FOUND)
2212 dio->range.n_sectors = ic->sectors_per_block;
2213 wait_and_add_new_range(ic, &dio->range);
2214 /*
2215 * wait_and_add_new_range drops the spinlock, so the journal
2216 * may have been changed arbitrarily. We need to recheck.
2217 * To simplify the code, we restrict I/O size to just one block.
2218 */
2219 if (journal_read_pos != NOT_FOUND) {
2220 sector_t next_sector;
2221 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2222 if (unlikely(new_pos != journal_read_pos)) {
2223 remove_range_unlocked(ic, &dio->range);
2224 goto retry;
2225 }
2226 }
2227 }
2228 if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {
2229 sector_t next_sector;
2230 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2231 if (unlikely(new_pos != NOT_FOUND) ||
2232 unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {
2233 remove_range_unlocked(ic, &dio->range);
2234 spin_unlock_irq(&ic->endio_wait.lock);
2235 queue_work(ic->commit_wq, &ic->commit_work);
2236 flush_workqueue(ic->commit_wq);
2237 queue_work(ic->writer_wq, &ic->writer_work);
2238 flush_workqueue(ic->writer_wq);
2239 discard_retried = true;
2240 goto lock_retry;
2241 }
2242 }
2243 spin_unlock_irq(&ic->endio_wait.lock);
2244
2245 if (unlikely(journal_read_pos != NOT_FOUND)) {
2246 journal_section = journal_read_pos / ic->journal_section_entries;
2247 journal_entry = journal_read_pos % ic->journal_section_entries;
2248 goto journal_read_write;
2249 }
2250
2251 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {
2252 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
2253 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
2254 struct bitmap_block_status *bbs;
2255
2256 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);
2257 spin_lock(&bbs->bio_queue_lock);
2258 bio_list_add(&bbs->bio_queue, bio);
2259 spin_unlock(&bbs->bio_queue_lock);
2260 queue_work(ic->writer_wq, &bbs->work);
2261 return;
2262 }
2263 }
2264
2265 dio->in_flight = (atomic_t)ATOMIC_INIT(2);
2266
2267 if (need_sync_io) {
2268 init_completion(&read_comp);
2269 dio->completion = &read_comp;
2270 } else
2271 dio->completion = NULL;
2272
2273 dm_bio_record(&dio->bio_details, bio);
2274 bio_set_dev(bio, ic->dev->bdev);
2275 bio->bi_integrity = NULL;
2276 bio->bi_opf &= ~REQ_INTEGRITY;
2277 bio->bi_end_io = integrity_end_io;
2278 bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
2279
2280 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {
2281 integrity_metadata(&dio->work);
2282 dm_integrity_flush_buffers(ic, false);
2283
2284 dio->in_flight = (atomic_t)ATOMIC_INIT(1);
2285 dio->completion = NULL;
2286
2287 submit_bio_noacct(bio);
2288
2289 return;
2290 }
2291
2292 submit_bio_noacct(bio);
2293
2294 if (need_sync_io) {
2295 wait_for_completion_io(&read_comp);
2296 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
2297 dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))
2298 goto skip_check;
2299 if (ic->mode == 'B') {
2300 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,
2301 dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))
2302 goto skip_check;
2303 }
2304
2305 if (likely(!bio->bi_status))
2306 integrity_metadata(&dio->work);
2307 else
2308skip_check:
2309 dec_in_flight(dio);
2310
2311 } else {
2312 INIT_WORK(&dio->work, integrity_metadata);
2313 queue_work(ic->metadata_wq, &dio->work);
2314 }
2315
2316 return;
2317
2318journal_read_write:
2319 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))
2320 goto lock_retry;
2321
2322 do_endio_flush(ic, dio);
2323}
2324
2325
2326static void integrity_bio_wait(struct work_struct *w)
2327{
2328 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
2329
2330 dm_integrity_map_continue(dio, false);
2331}
2332
2333static void pad_uncommitted(struct dm_integrity_c *ic)
2334{
2335 if (ic->free_section_entry) {
2336 ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry;
2337 ic->free_section_entry = 0;
2338 ic->free_section++;
2339 wraparound_section(ic, &ic->free_section);
2340 ic->n_uncommitted_sections++;
2341 }
2342 if (WARN_ON(ic->journal_sections * ic->journal_section_entries !=
2343 (ic->n_uncommitted_sections + ic->n_committed_sections) *
2344 ic->journal_section_entries + ic->free_sectors)) {
2345 DMCRIT("journal_sections %u, journal_section_entries %u, "
2346 "n_uncommitted_sections %u, n_committed_sections %u, "
2347 "journal_section_entries %u, free_sectors %u",
2348 ic->journal_sections, ic->journal_section_entries,
2349 ic->n_uncommitted_sections, ic->n_committed_sections,
2350 ic->journal_section_entries, ic->free_sectors);
2351 }
2352}
2353
2354static void integrity_commit(struct work_struct *w)
2355{
2356 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work);
2357 unsigned commit_start, commit_sections;
2358 unsigned i, j, n;
2359 struct bio *flushes;
2360
2361 del_timer(&ic->autocommit_timer);
2362
2363 spin_lock_irq(&ic->endio_wait.lock);
2364 flushes = bio_list_get(&ic->flush_bio_list);
2365 if (unlikely(ic->mode != 'J')) {
2366 spin_unlock_irq(&ic->endio_wait.lock);
2367 dm_integrity_flush_buffers(ic, true);
2368 goto release_flush_bios;
2369 }
2370
2371 pad_uncommitted(ic);
2372 commit_start = ic->uncommitted_section;
2373 commit_sections = ic->n_uncommitted_sections;
2374 spin_unlock_irq(&ic->endio_wait.lock);
2375
2376 if (!commit_sections)
2377 goto release_flush_bios;
2378
2379 ic->wrote_to_journal = true;
2380
2381 i = commit_start;
2382 for (n = 0; n < commit_sections; n++) {
2383 for (j = 0; j < ic->journal_section_entries; j++) {
2384 struct journal_entry *je;
2385 je = access_journal_entry(ic, i, j);
2386 io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
2387 }
2388 for (j = 0; j < ic->journal_section_sectors; j++) {
2389 struct journal_sector *js;
2390 js = access_journal(ic, i, j);
2391 js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq);
2392 }
2393 i++;
2394 if (unlikely(i >= ic->journal_sections))
2395 ic->commit_seq = next_commit_seq(ic->commit_seq);
2396 wraparound_section(ic, &i);
2397 }
2398 smp_rmb();
2399
2400 write_journal(ic, commit_start, commit_sections);
2401
2402 spin_lock_irq(&ic->endio_wait.lock);
2403 ic->uncommitted_section += commit_sections;
2404 wraparound_section(ic, &ic->uncommitted_section);
2405 ic->n_uncommitted_sections -= commit_sections;
2406 ic->n_committed_sections += commit_sections;
2407 spin_unlock_irq(&ic->endio_wait.lock);
2408
2409 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold)
2410 queue_work(ic->writer_wq, &ic->writer_work);
2411
2412release_flush_bios:
2413 while (flushes) {
2414 struct bio *next = flushes->bi_next;
2415 flushes->bi_next = NULL;
2416 do_endio(ic, flushes);
2417 flushes = next;
2418 }
2419}
2420
2421static void complete_copy_from_journal(unsigned long error, void *context)
2422{
2423 struct journal_io *io = context;
2424 struct journal_completion *comp = io->comp;
2425 struct dm_integrity_c *ic = comp->ic;
2426 remove_range(ic, &io->range);
2427 mempool_free(io, &ic->journal_io_mempool);
2428 if (unlikely(error != 0))
2429 dm_integrity_io_error(ic, "copying from journal", -EIO);
2430 complete_journal_op(comp);
2431}
2432
2433static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js,
2434 struct journal_entry *je)
2435{
2436 unsigned s = 0;
2437 do {
2438 js->commit_id = je->last_bytes[s];
2439 js++;
2440 } while (++s < ic->sectors_per_block);
2441}
2442
2443static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
2444 unsigned write_sections, bool from_replay)
2445{
2446 unsigned i, j, n;
2447 struct journal_completion comp;
2448 struct blk_plug plug;
2449
2450 blk_start_plug(&plug);
2451
2452 comp.ic = ic;
2453 comp.in_flight = (atomic_t)ATOMIC_INIT(1);
2454 init_completion(&comp.comp);
2455
2456 i = write_start;
2457 for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) {
2458#ifndef INTERNAL_VERIFY
2459 if (unlikely(from_replay))
2460#endif
2461 rw_section_mac(ic, i, false);
2462 for (j = 0; j < ic->journal_section_entries; j++) {
2463 struct journal_entry *je = access_journal_entry(ic, i, j);
2464 sector_t sec, area, offset;
2465 unsigned k, l, next_loop;
2466 sector_t metadata_block;
2467 unsigned metadata_offset;
2468 struct journal_io *io;
2469
2470 if (journal_entry_is_unused(je))
2471 continue;
2472 BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay);
2473 sec = journal_entry_get_sector(je);
2474 if (unlikely(from_replay)) {
2475 if (unlikely(sec & (unsigned)(ic->sectors_per_block - 1))) {
2476 dm_integrity_io_error(ic, "invalid sector in journal", -EIO);
2477 sec &= ~(sector_t)(ic->sectors_per_block - 1);
2478 }
2479 if (unlikely(sec >= ic->provided_data_sectors)) {
2480 journal_entry_set_unused(je);
2481 continue;
2482 }
2483 }
2484 get_area_and_offset(ic, sec, &area, &offset);
2485 restore_last_bytes(ic, access_journal_data(ic, i, j), je);
2486 for (k = j + 1; k < ic->journal_section_entries; k++) {
2487 struct journal_entry *je2 = access_journal_entry(ic, i, k);
2488 sector_t sec2, area2, offset2;
2489 if (journal_entry_is_unused(je2))
2490 break;
2491 BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay);
2492 sec2 = journal_entry_get_sector(je2);
2493 if (unlikely(sec2 >= ic->provided_data_sectors))
2494 break;
2495 get_area_and_offset(ic, sec2, &area2, &offset2);
2496 if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block))
2497 break;
2498 restore_last_bytes(ic, access_journal_data(ic, i, k), je2);
2499 }
2500 next_loop = k - 1;
2501
2502 io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO);
2503 io->comp = ∁
2504 io->range.logical_sector = sec;
2505 io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block;
2506
2507 spin_lock_irq(&ic->endio_wait.lock);
2508 add_new_range_and_wait(ic, &io->range);
2509
2510 if (likely(!from_replay)) {
2511 struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries];
2512
2513 /* don't write if there is newer committed sector */
2514 while (j < k && find_newer_committed_node(ic, §ion_node[j])) {
2515 struct journal_entry *je2 = access_journal_entry(ic, i, j);
2516
2517 journal_entry_set_unused(je2);
2518 remove_journal_node(ic, §ion_node[j]);
2519 j++;
2520 sec += ic->sectors_per_block;
2521 offset += ic->sectors_per_block;
2522 }
2523 while (j < k && find_newer_committed_node(ic, §ion_node[k - 1])) {
2524 struct journal_entry *je2 = access_journal_entry(ic, i, k - 1);
2525
2526 journal_entry_set_unused(je2);
2527 remove_journal_node(ic, §ion_node[k - 1]);
2528 k--;
2529 }
2530 if (j == k) {
2531 remove_range_unlocked(ic, &io->range);
2532 spin_unlock_irq(&ic->endio_wait.lock);
2533 mempool_free(io, &ic->journal_io_mempool);
2534 goto skip_io;
2535 }
2536 for (l = j; l < k; l++) {
2537 remove_journal_node(ic, §ion_node[l]);
2538 }
2539 }
2540 spin_unlock_irq(&ic->endio_wait.lock);
2541
2542 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
2543 for (l = j; l < k; l++) {
2544 int r;
2545 struct journal_entry *je2 = access_journal_entry(ic, i, l);
2546
2547 if (
2548#ifndef INTERNAL_VERIFY
2549 unlikely(from_replay) &&
2550#endif
2551 ic->internal_hash) {
2552 char test_tag[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
2553
2554 integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block),
2555 (char *)access_journal_data(ic, i, l), test_tag);
2556 if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) {
2557 dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ);
2558 dm_audit_log_target(DM_MSG_PREFIX, "integrity-replay-journal", ic->ti, 0);
2559 }
2560 }
2561
2562 journal_entry_set_unused(je2);
2563 r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset,
2564 ic->tag_size, TAG_WRITE);
2565 if (unlikely(r)) {
2566 dm_integrity_io_error(ic, "reading tags", r);
2567 }
2568 }
2569
2570 atomic_inc(&comp.in_flight);
2571 copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block,
2572 (k - j) << ic->sb->log2_sectors_per_block,
2573 get_data_sector(ic, area, offset),
2574 complete_copy_from_journal, io);
2575skip_io:
2576 j = next_loop;
2577 }
2578 }
2579
2580 dm_bufio_write_dirty_buffers_async(ic->bufio);
2581
2582 blk_finish_plug(&plug);
2583
2584 complete_journal_op(&comp);
2585 wait_for_completion_io(&comp.comp);
2586
2587 dm_integrity_flush_buffers(ic, true);
2588}
2589
2590static void integrity_writer(struct work_struct *w)
2591{
2592 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work);
2593 unsigned write_start, write_sections;
2594
2595 unsigned prev_free_sectors;
2596
2597 spin_lock_irq(&ic->endio_wait.lock);
2598 write_start = ic->committed_section;
2599 write_sections = ic->n_committed_sections;
2600 spin_unlock_irq(&ic->endio_wait.lock);
2601
2602 if (!write_sections)
2603 return;
2604
2605 do_journal_write(ic, write_start, write_sections, false);
2606
2607 spin_lock_irq(&ic->endio_wait.lock);
2608
2609 ic->committed_section += write_sections;
2610 wraparound_section(ic, &ic->committed_section);
2611 ic->n_committed_sections -= write_sections;
2612
2613 prev_free_sectors = ic->free_sectors;
2614 ic->free_sectors += write_sections * ic->journal_section_entries;
2615 if (unlikely(!prev_free_sectors))
2616 wake_up_locked(&ic->endio_wait);
2617
2618 spin_unlock_irq(&ic->endio_wait.lock);
2619}
2620
2621static void recalc_write_super(struct dm_integrity_c *ic)
2622{
2623 int r;
2624
2625 dm_integrity_flush_buffers(ic, false);
2626 if (dm_integrity_failed(ic))
2627 return;
2628
2629 r = sync_rw_sb(ic, REQ_OP_WRITE);
2630 if (unlikely(r))
2631 dm_integrity_io_error(ic, "writing superblock", r);
2632}
2633
2634static void integrity_recalc(struct work_struct *w)
2635{
2636 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work);
2637 struct dm_integrity_range range;
2638 struct dm_io_request io_req;
2639 struct dm_io_region io_loc;
2640 sector_t area, offset;
2641 sector_t metadata_block;
2642 unsigned metadata_offset;
2643 sector_t logical_sector, n_sectors;
2644 __u8 *t;
2645 unsigned i;
2646 int r;
2647 unsigned super_counter = 0;
2648
2649 DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector));
2650
2651 spin_lock_irq(&ic->endio_wait.lock);
2652
2653next_chunk:
2654
2655 if (unlikely(dm_post_suspending(ic->ti)))
2656 goto unlock_ret;
2657
2658 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
2659 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) {
2660 if (ic->mode == 'B') {
2661 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
2662 DEBUG_print("queue_delayed_work: bitmap_flush_work\n");
2663 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
2664 }
2665 goto unlock_ret;
2666 }
2667
2668 get_area_and_offset(ic, range.logical_sector, &area, &offset);
2669 range.n_sectors = min((sector_t)RECALC_SECTORS, ic->provided_data_sectors - range.logical_sector);
2670 if (!ic->meta_dev)
2671 range.n_sectors = min(range.n_sectors, ((sector_t)1U << ic->sb->log2_interleave_sectors) - (unsigned)offset);
2672
2673 add_new_range_and_wait(ic, &range);
2674 spin_unlock_irq(&ic->endio_wait.lock);
2675 logical_sector = range.logical_sector;
2676 n_sectors = range.n_sectors;
2677
2678 if (ic->mode == 'B') {
2679 if (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) {
2680 goto advance_and_next;
2681 }
2682 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector,
2683 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) {
2684 logical_sector += ic->sectors_per_block;
2685 n_sectors -= ic->sectors_per_block;
2686 cond_resched();
2687 }
2688 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector + n_sectors - ic->sectors_per_block,
2689 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) {
2690 n_sectors -= ic->sectors_per_block;
2691 cond_resched();
2692 }
2693 get_area_and_offset(ic, logical_sector, &area, &offset);
2694 }
2695
2696 DEBUG_print("recalculating: %llx, %llx\n", logical_sector, n_sectors);
2697
2698 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) {
2699 recalc_write_super(ic);
2700 if (ic->mode == 'B') {
2701 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval);
2702 }
2703 super_counter = 0;
2704 }
2705
2706 if (unlikely(dm_integrity_failed(ic)))
2707 goto err;
2708
2709 io_req.bi_opf = REQ_OP_READ;
2710 io_req.mem.type = DM_IO_VMA;
2711 io_req.mem.ptr.addr = ic->recalc_buffer;
2712 io_req.notify.fn = NULL;
2713 io_req.client = ic->io;
2714 io_loc.bdev = ic->dev->bdev;
2715 io_loc.sector = get_data_sector(ic, area, offset);
2716 io_loc.count = n_sectors;
2717
2718 r = dm_io(&io_req, 1, &io_loc, NULL);
2719 if (unlikely(r)) {
2720 dm_integrity_io_error(ic, "reading data", r);
2721 goto err;
2722 }
2723
2724 t = ic->recalc_tags;
2725 for (i = 0; i < n_sectors; i += ic->sectors_per_block) {
2726 integrity_sector_checksum(ic, logical_sector + i, ic->recalc_buffer + (i << SECTOR_SHIFT), t);
2727 t += ic->tag_size;
2728 }
2729
2730 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
2731
2732 r = dm_integrity_rw_tag(ic, ic->recalc_tags, &metadata_block, &metadata_offset, t - ic->recalc_tags, TAG_WRITE);
2733 if (unlikely(r)) {
2734 dm_integrity_io_error(ic, "writing tags", r);
2735 goto err;
2736 }
2737
2738 if (ic->mode == 'B') {
2739 sector_t start, end;
2740 start = (range.logical_sector >>
2741 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) <<
2742 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
2743 end = ((range.logical_sector + range.n_sectors) >>
2744 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) <<
2745 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
2746 block_bitmap_op(ic, ic->recalc_bitmap, start, end - start, BITMAP_OP_CLEAR);
2747 }
2748
2749advance_and_next:
2750 cond_resched();
2751
2752 spin_lock_irq(&ic->endio_wait.lock);
2753 remove_range_unlocked(ic, &range);
2754 ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors);
2755 goto next_chunk;
2756
2757err:
2758 remove_range(ic, &range);
2759 return;
2760
2761unlock_ret:
2762 spin_unlock_irq(&ic->endio_wait.lock);
2763
2764 recalc_write_super(ic);
2765}
2766
2767static void bitmap_block_work(struct work_struct *w)
2768{
2769 struct bitmap_block_status *bbs = container_of(w, struct bitmap_block_status, work);
2770 struct dm_integrity_c *ic = bbs->ic;
2771 struct bio *bio;
2772 struct bio_list bio_queue;
2773 struct bio_list waiting;
2774
2775 bio_list_init(&waiting);
2776
2777 spin_lock(&bbs->bio_queue_lock);
2778 bio_queue = bbs->bio_queue;
2779 bio_list_init(&bbs->bio_queue);
2780 spin_unlock(&bbs->bio_queue_lock);
2781
2782 while ((bio = bio_list_pop(&bio_queue))) {
2783 struct dm_integrity_io *dio;
2784
2785 dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
2786
2787 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
2788 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
2789 remove_range(ic, &dio->range);
2790 INIT_WORK(&dio->work, integrity_bio_wait);
2791 queue_work(ic->offload_wq, &dio->work);
2792 } else {
2793 block_bitmap_op(ic, ic->journal, dio->range.logical_sector,
2794 dio->range.n_sectors, BITMAP_OP_SET);
2795 bio_list_add(&waiting, bio);
2796 }
2797 }
2798
2799 if (bio_list_empty(&waiting))
2800 return;
2801
2802 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC,
2803 bbs->idx * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT),
2804 BITMAP_BLOCK_SIZE >> SECTOR_SHIFT, NULL);
2805
2806 while ((bio = bio_list_pop(&waiting))) {
2807 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
2808
2809 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
2810 dio->range.n_sectors, BITMAP_OP_SET);
2811
2812 remove_range(ic, &dio->range);
2813 INIT_WORK(&dio->work, integrity_bio_wait);
2814 queue_work(ic->offload_wq, &dio->work);
2815 }
2816
2817 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval);
2818}
2819
2820static void bitmap_flush_work(struct work_struct *work)
2821{
2822 struct dm_integrity_c *ic = container_of(work, struct dm_integrity_c, bitmap_flush_work.work);
2823 struct dm_integrity_range range;
2824 unsigned long limit;
2825 struct bio *bio;
2826
2827 dm_integrity_flush_buffers(ic, false);
2828
2829 range.logical_sector = 0;
2830 range.n_sectors = ic->provided_data_sectors;
2831
2832 spin_lock_irq(&ic->endio_wait.lock);
2833 add_new_range_and_wait(ic, &range);
2834 spin_unlock_irq(&ic->endio_wait.lock);
2835
2836 dm_integrity_flush_buffers(ic, true);
2837
2838 limit = ic->provided_data_sectors;
2839 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
2840 limit = le64_to_cpu(ic->sb->recalc_sector)
2841 >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)
2842 << (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
2843 }
2844 /*DEBUG_print("zeroing journal\n");*/
2845 block_bitmap_op(ic, ic->journal, 0, limit, BITMAP_OP_CLEAR);
2846 block_bitmap_op(ic, ic->may_write_bitmap, 0, limit, BITMAP_OP_CLEAR);
2847
2848 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
2849 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
2850
2851 spin_lock_irq(&ic->endio_wait.lock);
2852 remove_range_unlocked(ic, &range);
2853 while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) {
2854 bio_endio(bio);
2855 spin_unlock_irq(&ic->endio_wait.lock);
2856 spin_lock_irq(&ic->endio_wait.lock);
2857 }
2858 spin_unlock_irq(&ic->endio_wait.lock);
2859}
2860
2861
2862static void init_journal(struct dm_integrity_c *ic, unsigned start_section,
2863 unsigned n_sections, unsigned char commit_seq)
2864{
2865 unsigned i, j, n;
2866
2867 if (!n_sections)
2868 return;
2869
2870 for (n = 0; n < n_sections; n++) {
2871 i = start_section + n;
2872 wraparound_section(ic, &i);
2873 for (j = 0; j < ic->journal_section_sectors; j++) {
2874 struct journal_sector *js = access_journal(ic, i, j);
2875 BUILD_BUG_ON(sizeof(js->sectors) != JOURNAL_SECTOR_DATA);
2876 memset(&js->sectors, 0, sizeof(js->sectors));
2877 js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq);
2878 }
2879 for (j = 0; j < ic->journal_section_entries; j++) {
2880 struct journal_entry *je = access_journal_entry(ic, i, j);
2881 journal_entry_set_unused(je);
2882 }
2883 }
2884
2885 write_journal(ic, start_section, n_sections);
2886}
2887
2888static int find_commit_seq(struct dm_integrity_c *ic, unsigned i, unsigned j, commit_id_t id)
2889{
2890 unsigned char k;
2891 for (k = 0; k < N_COMMIT_IDS; k++) {
2892 if (dm_integrity_commit_id(ic, i, j, k) == id)
2893 return k;
2894 }
2895 dm_integrity_io_error(ic, "journal commit id", -EIO);
2896 return -EIO;
2897}
2898
2899static void replay_journal(struct dm_integrity_c *ic)
2900{
2901 unsigned i, j;
2902 bool used_commit_ids[N_COMMIT_IDS];
2903 unsigned max_commit_id_sections[N_COMMIT_IDS];
2904 unsigned write_start, write_sections;
2905 unsigned continue_section;
2906 bool journal_empty;
2907 unsigned char unused, last_used, want_commit_seq;
2908
2909 if (ic->mode == 'R')
2910 return;
2911
2912 if (ic->journal_uptodate)
2913 return;
2914
2915 last_used = 0;
2916 write_start = 0;
2917
2918 if (!ic->just_formatted) {
2919 DEBUG_print("reading journal\n");
2920 rw_journal(ic, REQ_OP_READ, 0, ic->journal_sections, NULL);
2921 if (ic->journal_io)
2922 DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal");
2923 if (ic->journal_io) {
2924 struct journal_completion crypt_comp;
2925 crypt_comp.ic = ic;
2926 init_completion(&crypt_comp.comp);
2927 crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0);
2928 encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp);
2929 wait_for_completion(&crypt_comp.comp);
2930 }
2931 DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal");
2932 }
2933
2934 if (dm_integrity_failed(ic))
2935 goto clear_journal;
2936
2937 journal_empty = true;
2938 memset(used_commit_ids, 0, sizeof used_commit_ids);
2939 memset(max_commit_id_sections, 0, sizeof max_commit_id_sections);
2940 for (i = 0; i < ic->journal_sections; i++) {
2941 for (j = 0; j < ic->journal_section_sectors; j++) {
2942 int k;
2943 struct journal_sector *js = access_journal(ic, i, j);
2944 k = find_commit_seq(ic, i, j, js->commit_id);
2945 if (k < 0)
2946 goto clear_journal;
2947 used_commit_ids[k] = true;
2948 max_commit_id_sections[k] = i;
2949 }
2950 if (journal_empty) {
2951 for (j = 0; j < ic->journal_section_entries; j++) {
2952 struct journal_entry *je = access_journal_entry(ic, i, j);
2953 if (!journal_entry_is_unused(je)) {
2954 journal_empty = false;
2955 break;
2956 }
2957 }
2958 }
2959 }
2960
2961 if (!used_commit_ids[N_COMMIT_IDS - 1]) {
2962 unused = N_COMMIT_IDS - 1;
2963 while (unused && !used_commit_ids[unused - 1])
2964 unused--;
2965 } else {
2966 for (unused = 0; unused < N_COMMIT_IDS; unused++)
2967 if (!used_commit_ids[unused])
2968 break;
2969 if (unused == N_COMMIT_IDS) {
2970 dm_integrity_io_error(ic, "journal commit ids", -EIO);
2971 goto clear_journal;
2972 }
2973 }
2974 DEBUG_print("first unused commit seq %d [%d,%d,%d,%d]\n",
2975 unused, used_commit_ids[0], used_commit_ids[1],
2976 used_commit_ids[2], used_commit_ids[3]);
2977
2978 last_used = prev_commit_seq(unused);
2979 want_commit_seq = prev_commit_seq(last_used);
2980
2981 if (!used_commit_ids[want_commit_seq] && used_commit_ids[prev_commit_seq(want_commit_seq)])
2982 journal_empty = true;
2983
2984 write_start = max_commit_id_sections[last_used] + 1;
2985 if (unlikely(write_start >= ic->journal_sections))
2986 want_commit_seq = next_commit_seq(want_commit_seq);
2987 wraparound_section(ic, &write_start);
2988
2989 i = write_start;
2990 for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) {
2991 for (j = 0; j < ic->journal_section_sectors; j++) {
2992 struct journal_sector *js = access_journal(ic, i, j);
2993
2994 if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) {
2995 /*
2996 * This could be caused by crash during writing.
2997 * We won't replay the inconsistent part of the
2998 * journal.
2999 */
3000 DEBUG_print("commit id mismatch at position (%u, %u): %d != %d\n",
3001 i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq);
3002 goto brk;
3003 }
3004 }
3005 i++;
3006 if (unlikely(i >= ic->journal_sections))
3007 want_commit_seq = next_commit_seq(want_commit_seq);
3008 wraparound_section(ic, &i);
3009 }
3010brk:
3011
3012 if (!journal_empty) {
3013 DEBUG_print("replaying %u sections, starting at %u, commit seq %d\n",
3014 write_sections, write_start, want_commit_seq);
3015 do_journal_write(ic, write_start, write_sections, true);
3016 }
3017
3018 if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) {
3019 continue_section = write_start;
3020 ic->commit_seq = want_commit_seq;
3021 DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq);
3022 } else {
3023 unsigned s;
3024 unsigned char erase_seq;
3025clear_journal:
3026 DEBUG_print("clearing journal\n");
3027
3028 erase_seq = prev_commit_seq(prev_commit_seq(last_used));
3029 s = write_start;
3030 init_journal(ic, s, 1, erase_seq);
3031 s++;
3032 wraparound_section(ic, &s);
3033 if (ic->journal_sections >= 2) {
3034 init_journal(ic, s, ic->journal_sections - 2, erase_seq);
3035 s += ic->journal_sections - 2;
3036 wraparound_section(ic, &s);
3037 init_journal(ic, s, 1, erase_seq);
3038 }
3039
3040 continue_section = 0;
3041 ic->commit_seq = next_commit_seq(erase_seq);
3042 }
3043
3044 ic->committed_section = continue_section;
3045 ic->n_committed_sections = 0;
3046
3047 ic->uncommitted_section = continue_section;
3048 ic->n_uncommitted_sections = 0;
3049
3050 ic->free_section = continue_section;
3051 ic->free_section_entry = 0;
3052 ic->free_sectors = ic->journal_entries;
3053
3054 ic->journal_tree_root = RB_ROOT;
3055 for (i = 0; i < ic->journal_entries; i++)
3056 init_journal_node(&ic->journal_tree[i]);
3057}
3058
3059static void dm_integrity_enter_synchronous_mode(struct dm_integrity_c *ic)
3060{
3061 DEBUG_print("dm_integrity_enter_synchronous_mode\n");
3062
3063 if (ic->mode == 'B') {
3064 ic->bitmap_flush_interval = msecs_to_jiffies(10) + 1;
3065 ic->synchronous_mode = 1;
3066
3067 cancel_delayed_work_sync(&ic->bitmap_flush_work);
3068 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
3069 flush_workqueue(ic->commit_wq);
3070 }
3071}
3072
3073static int dm_integrity_reboot(struct notifier_block *n, unsigned long code, void *x)
3074{
3075 struct dm_integrity_c *ic = container_of(n, struct dm_integrity_c, reboot_notifier);
3076
3077 DEBUG_print("dm_integrity_reboot\n");
3078
3079 dm_integrity_enter_synchronous_mode(ic);
3080
3081 return NOTIFY_DONE;
3082}
3083
3084static void dm_integrity_postsuspend(struct dm_target *ti)
3085{
3086 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
3087 int r;
3088
3089 WARN_ON(unregister_reboot_notifier(&ic->reboot_notifier));
3090
3091 del_timer_sync(&ic->autocommit_timer);
3092
3093 if (ic->recalc_wq)
3094 drain_workqueue(ic->recalc_wq);
3095
3096 if (ic->mode == 'B')
3097 cancel_delayed_work_sync(&ic->bitmap_flush_work);
3098
3099 queue_work(ic->commit_wq, &ic->commit_work);
3100 drain_workqueue(ic->commit_wq);
3101
3102 if (ic->mode == 'J') {
3103 queue_work(ic->writer_wq, &ic->writer_work);
3104 drain_workqueue(ic->writer_wq);
3105 dm_integrity_flush_buffers(ic, true);
3106 if (ic->wrote_to_journal) {
3107 init_journal(ic, ic->free_section,
3108 ic->journal_sections - ic->free_section, ic->commit_seq);
3109 if (ic->free_section) {
3110 init_journal(ic, 0, ic->free_section,
3111 next_commit_seq(ic->commit_seq));
3112 }
3113 }
3114 }
3115
3116 if (ic->mode == 'B') {
3117 dm_integrity_flush_buffers(ic, true);
3118#if 1
3119 /* set to 0 to test bitmap replay code */
3120 init_journal(ic, 0, ic->journal_sections, 0);
3121 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3122 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3123 if (unlikely(r))
3124 dm_integrity_io_error(ic, "writing superblock", r);
3125#endif
3126 }
3127
3128 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
3129
3130 ic->journal_uptodate = true;
3131}
3132
3133static void dm_integrity_resume(struct dm_target *ti)
3134{
3135 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
3136 __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors);
3137 int r;
3138
3139 DEBUG_print("resume\n");
3140
3141 ic->wrote_to_journal = false;
3142
3143 if (ic->provided_data_sectors != old_provided_data_sectors) {
3144 if (ic->provided_data_sectors > old_provided_data_sectors &&
3145 ic->mode == 'B' &&
3146 ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) {
3147 rw_journal_sectors(ic, REQ_OP_READ, 0,
3148 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3149 block_bitmap_op(ic, ic->journal, old_provided_data_sectors,
3150 ic->provided_data_sectors - old_provided_data_sectors, BITMAP_OP_SET);
3151 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3152 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3153 }
3154
3155 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
3156 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3157 if (unlikely(r))
3158 dm_integrity_io_error(ic, "writing superblock", r);
3159 }
3160
3161 if (ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) {
3162 DEBUG_print("resume dirty_bitmap\n");
3163 rw_journal_sectors(ic, REQ_OP_READ, 0,
3164 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3165 if (ic->mode == 'B') {
3166 if (ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit &&
3167 !ic->reset_recalculate_flag) {
3168 block_bitmap_copy(ic, ic->recalc_bitmap, ic->journal);
3169 block_bitmap_copy(ic, ic->may_write_bitmap, ic->journal);
3170 if (!block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors,
3171 BITMAP_OP_TEST_ALL_CLEAR)) {
3172 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3173 ic->sb->recalc_sector = cpu_to_le64(0);
3174 }
3175 } else {
3176 DEBUG_print("non-matching blocks_per_bitmap_bit: %u, %u\n",
3177 ic->sb->log2_blocks_per_bitmap_bit, ic->log2_blocks_per_bitmap_bit);
3178 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit;
3179 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3180 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3181 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3182 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3183 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3184 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3185 ic->sb->recalc_sector = cpu_to_le64(0);
3186 }
3187 } else {
3188 if (!(ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit &&
3189 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_TEST_ALL_CLEAR)) ||
3190 ic->reset_recalculate_flag) {
3191 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3192 ic->sb->recalc_sector = cpu_to_le64(0);
3193 }
3194 init_journal(ic, 0, ic->journal_sections, 0);
3195 replay_journal(ic);
3196 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3197 }
3198 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3199 if (unlikely(r))
3200 dm_integrity_io_error(ic, "writing superblock", r);
3201 } else {
3202 replay_journal(ic);
3203 if (ic->reset_recalculate_flag) {
3204 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3205 ic->sb->recalc_sector = cpu_to_le64(0);
3206 }
3207 if (ic->mode == 'B') {
3208 ic->sb->flags |= cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3209 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit;
3210 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3211 if (unlikely(r))
3212 dm_integrity_io_error(ic, "writing superblock", r);
3213
3214 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3215 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3216 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3217 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
3218 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors) {
3219 block_bitmap_op(ic, ic->journal, le64_to_cpu(ic->sb->recalc_sector),
3220 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3221 block_bitmap_op(ic, ic->recalc_bitmap, le64_to_cpu(ic->sb->recalc_sector),
3222 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3223 block_bitmap_op(ic, ic->may_write_bitmap, le64_to_cpu(ic->sb->recalc_sector),
3224 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3225 }
3226 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3227 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3228 }
3229 }
3230
3231 DEBUG_print("testing recalc: %x\n", ic->sb->flags);
3232 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
3233 __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector);
3234 DEBUG_print("recalc pos: %llx / %llx\n", recalc_pos, ic->provided_data_sectors);
3235 if (recalc_pos < ic->provided_data_sectors) {
3236 queue_work(ic->recalc_wq, &ic->recalc_work);
3237 } else if (recalc_pos > ic->provided_data_sectors) {
3238 ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors);
3239 recalc_write_super(ic);
3240 }
3241 }
3242
3243 ic->reboot_notifier.notifier_call = dm_integrity_reboot;
3244 ic->reboot_notifier.next = NULL;
3245 ic->reboot_notifier.priority = INT_MAX - 1; /* be notified after md and before hardware drivers */
3246 WARN_ON(register_reboot_notifier(&ic->reboot_notifier));
3247
3248#if 0
3249 /* set to 1 to stress test synchronous mode */
3250 dm_integrity_enter_synchronous_mode(ic);
3251#endif
3252}
3253
3254static void dm_integrity_status(struct dm_target *ti, status_type_t type,
3255 unsigned status_flags, char *result, unsigned maxlen)
3256{
3257 struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
3258 unsigned arg_count;
3259 size_t sz = 0;
3260
3261 switch (type) {
3262 case STATUSTYPE_INFO:
3263 DMEMIT("%llu %llu",
3264 (unsigned long long)atomic64_read(&ic->number_of_mismatches),
3265 ic->provided_data_sectors);
3266 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
3267 DMEMIT(" %llu", le64_to_cpu(ic->sb->recalc_sector));
3268 else
3269 DMEMIT(" -");
3270 break;
3271
3272 case STATUSTYPE_TABLE: {
3273 __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100;
3274 watermark_percentage += ic->journal_entries / 2;
3275 do_div(watermark_percentage, ic->journal_entries);
3276 arg_count = 3;
3277 arg_count += !!ic->meta_dev;
3278 arg_count += ic->sectors_per_block != 1;
3279 arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING));
3280 arg_count += ic->reset_recalculate_flag;
3281 arg_count += ic->discard;
3282 arg_count += ic->mode == 'J';
3283 arg_count += ic->mode == 'J';
3284 arg_count += ic->mode == 'B';
3285 arg_count += ic->mode == 'B';
3286 arg_count += !!ic->internal_hash_alg.alg_string;
3287 arg_count += !!ic->journal_crypt_alg.alg_string;
3288 arg_count += !!ic->journal_mac_alg.alg_string;
3289 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0;
3290 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0;
3291 arg_count += ic->legacy_recalculate;
3292 DMEMIT("%s %llu %u %c %u", ic->dev->name, ic->start,
3293 ic->tag_size, ic->mode, arg_count);
3294 if (ic->meta_dev)
3295 DMEMIT(" meta_device:%s", ic->meta_dev->name);
3296 if (ic->sectors_per_block != 1)
3297 DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT);
3298 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
3299 DMEMIT(" recalculate");
3300 if (ic->reset_recalculate_flag)
3301 DMEMIT(" reset_recalculate");
3302 if (ic->discard)
3303 DMEMIT(" allow_discards");
3304 DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS);
3305 DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors);
3306 DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
3307 if (ic->mode == 'J') {
3308 DMEMIT(" journal_watermark:%u", (unsigned)watermark_percentage);
3309 DMEMIT(" commit_time:%u", ic->autocommit_msec);
3310 }
3311 if (ic->mode == 'B') {
3312 DMEMIT(" sectors_per_bit:%llu", (sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit);
3313 DMEMIT(" bitmap_flush_interval:%u", jiffies_to_msecs(ic->bitmap_flush_interval));
3314 }
3315 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0)
3316 DMEMIT(" fix_padding");
3317 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0)
3318 DMEMIT(" fix_hmac");
3319 if (ic->legacy_recalculate)
3320 DMEMIT(" legacy_recalculate");
3321
3322#define EMIT_ALG(a, n) \
3323 do { \
3324 if (ic->a.alg_string) { \
3325 DMEMIT(" %s:%s", n, ic->a.alg_string); \
3326 if (ic->a.key_string) \
3327 DMEMIT(":%s", ic->a.key_string);\
3328 } \
3329 } while (0)
3330 EMIT_ALG(internal_hash_alg, "internal_hash");
3331 EMIT_ALG(journal_crypt_alg, "journal_crypt");
3332 EMIT_ALG(journal_mac_alg, "journal_mac");
3333 break;
3334 }
3335 case STATUSTYPE_IMA:
3336 DMEMIT_TARGET_NAME_VERSION(ti->type);
3337 DMEMIT(",dev_name=%s,start=%llu,tag_size=%u,mode=%c",
3338 ic->dev->name, ic->start, ic->tag_size, ic->mode);
3339
3340 if (ic->meta_dev)
3341 DMEMIT(",meta_device=%s", ic->meta_dev->name);
3342 if (ic->sectors_per_block != 1)
3343 DMEMIT(",block_size=%u", ic->sectors_per_block << SECTOR_SHIFT);
3344
3345 DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ?
3346 'y' : 'n');
3347 DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n');
3348 DMEMIT(",fix_padding=%c",
3349 ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n');
3350 DMEMIT(",fix_hmac=%c",
3351 ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) ? 'y' : 'n');
3352 DMEMIT(",legacy_recalculate=%c", ic->legacy_recalculate ? 'y' : 'n');
3353
3354 DMEMIT(",journal_sectors=%u", ic->initial_sectors - SB_SECTORS);
3355 DMEMIT(",interleave_sectors=%u", 1U << ic->sb->log2_interleave_sectors);
3356 DMEMIT(",buffer_sectors=%u", 1U << ic->log2_buffer_sectors);
3357 DMEMIT(";");
3358 break;
3359 }
3360}
3361
3362static int dm_integrity_iterate_devices(struct dm_target *ti,
3363 iterate_devices_callout_fn fn, void *data)
3364{
3365 struct dm_integrity_c *ic = ti->private;
3366
3367 if (!ic->meta_dev)
3368 return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data);
3369 else
3370 return fn(ti, ic->dev, 0, ti->len, data);
3371}
3372
3373static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits)
3374{
3375 struct dm_integrity_c *ic = ti->private;
3376
3377 if (ic->sectors_per_block > 1) {
3378 limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
3379 limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
3380 blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT);
3381 limits->dma_alignment = limits->logical_block_size - 1;
3382 }
3383}
3384
3385static void calculate_journal_section_size(struct dm_integrity_c *ic)
3386{
3387 unsigned sector_space = JOURNAL_SECTOR_DATA;
3388
3389 ic->journal_sections = le32_to_cpu(ic->sb->journal_sections);
3390 ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size,
3391 JOURNAL_ENTRY_ROUNDUP);
3392
3393 if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC))
3394 sector_space -= JOURNAL_MAC_PER_SECTOR;
3395 ic->journal_entries_per_sector = sector_space / ic->journal_entry_size;
3396 ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS;
3397 ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JOURNAL_BLOCK_SECTORS;
3398 ic->journal_entries = ic->journal_section_entries * ic->journal_sections;
3399}
3400
3401static int calculate_device_limits(struct dm_integrity_c *ic)
3402{
3403 __u64 initial_sectors;
3404
3405 calculate_journal_section_size(ic);
3406 initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections;
3407 if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UINT_MAX)
3408 return -EINVAL;
3409 ic->initial_sectors = initial_sectors;
3410
3411 if (!ic->meta_dev) {
3412 sector_t last_sector, last_area, last_offset;
3413
3414 /* we have to maintain excessive padding for compatibility with existing volumes */
3415 __u64 metadata_run_padding =
3416 ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING) ?
3417 (__u64)(METADATA_PADDING_SECTORS << SECTOR_SHIFT) :
3418 (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS);
3419
3420 ic->metadata_run = round_up((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block),
3421 metadata_run_padding) >> SECTOR_SHIFT;
3422 if (!(ic->metadata_run & (ic->metadata_run - 1)))
3423 ic->log2_metadata_run = __ffs(ic->metadata_run);
3424 else
3425 ic->log2_metadata_run = -1;
3426
3427 get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset);
3428 last_sector = get_data_sector(ic, last_area, last_offset);
3429 if (last_sector < ic->start || last_sector >= ic->meta_device_sectors)
3430 return -EINVAL;
3431 } else {
3432 __u64 meta_size = (ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size;
3433 meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1))
3434 >> (ic->log2_buffer_sectors + SECTOR_SHIFT);
3435 meta_size <<= ic->log2_buffer_sectors;
3436 if (ic->initial_sectors + meta_size < ic->initial_sectors ||
3437 ic->initial_sectors + meta_size > ic->meta_device_sectors)
3438 return -EINVAL;
3439 ic->metadata_run = 1;
3440 ic->log2_metadata_run = 0;
3441 }
3442
3443 return 0;
3444}
3445
3446static void get_provided_data_sectors(struct dm_integrity_c *ic)
3447{
3448 if (!ic->meta_dev) {
3449 int test_bit;
3450 ic->provided_data_sectors = 0;
3451 for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) {
3452 __u64 prev_data_sectors = ic->provided_data_sectors;
3453
3454 ic->provided_data_sectors |= (sector_t)1 << test_bit;
3455 if (calculate_device_limits(ic))
3456 ic->provided_data_sectors = prev_data_sectors;
3457 }
3458 } else {
3459 ic->provided_data_sectors = ic->data_device_sectors;
3460 ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1);
3461 }
3462}
3463
3464static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sectors, unsigned interleave_sectors)
3465{
3466 unsigned journal_sections;
3467 int test_bit;
3468
3469 memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT);
3470 memcpy(ic->sb->magic, SB_MAGIC, 8);
3471 ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size);
3472 ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block);
3473 if (ic->journal_mac_alg.alg_string)
3474 ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC);
3475
3476 calculate_journal_section_size(ic);
3477 journal_sections = journal_sectors / ic->journal_section_sectors;
3478 if (!journal_sections)
3479 journal_sections = 1;
3480
3481 if (ic->fix_hmac && (ic->internal_hash_alg.alg_string || ic->journal_mac_alg.alg_string)) {
3482 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_HMAC);
3483 get_random_bytes(ic->sb->salt, SALT_SIZE);
3484 }
3485
3486 if (!ic->meta_dev) {
3487 if (ic->fix_padding)
3488 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING);
3489 ic->sb->journal_sections = cpu_to_le32(journal_sections);
3490 if (!interleave_sectors)
3491 interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
3492 ic->sb->log2_interleave_sectors = __fls(interleave_sectors);
3493 ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
3494 ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
3495
3496 get_provided_data_sectors(ic);
3497 if (!ic->provided_data_sectors)
3498 return -EINVAL;
3499 } else {
3500 ic->sb->log2_interleave_sectors = 0;
3501
3502 get_provided_data_sectors(ic);
3503 if (!ic->provided_data_sectors)
3504 return -EINVAL;
3505
3506try_smaller_buffer:
3507 ic->sb->journal_sections = cpu_to_le32(0);
3508 for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) {
3509 __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections);
3510 __u32 test_journal_sections = prev_journal_sections | (1U << test_bit);
3511 if (test_journal_sections > journal_sections)
3512 continue;
3513 ic->sb->journal_sections = cpu_to_le32(test_journal_sections);
3514 if (calculate_device_limits(ic))
3515 ic->sb->journal_sections = cpu_to_le32(prev_journal_sections);
3516
3517 }
3518 if (!le32_to_cpu(ic->sb->journal_sections)) {
3519 if (ic->log2_buffer_sectors > 3) {
3520 ic->log2_buffer_sectors--;
3521 goto try_smaller_buffer;
3522 }
3523 return -EINVAL;
3524 }
3525 }
3526
3527 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
3528
3529 sb_set_version(ic);
3530
3531 return 0;
3532}
3533
3534static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic)
3535{
3536 struct gendisk *disk = dm_disk(dm_table_get_md(ti->table));
3537 struct blk_integrity bi;
3538
3539 memset(&bi, 0, sizeof(bi));
3540 bi.profile = &dm_integrity_profile;
3541 bi.tuple_size = ic->tag_size;
3542 bi.tag_size = bi.tuple_size;
3543 bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT;
3544
3545 blk_integrity_register(disk, &bi);
3546 blk_queue_max_integrity_segments(disk->queue, UINT_MAX);
3547}
3548
3549static void dm_integrity_free_page_list(struct page_list *pl)
3550{
3551 unsigned i;
3552
3553 if (!pl)
3554 return;
3555 for (i = 0; pl[i].page; i++)
3556 __free_page(pl[i].page);
3557 kvfree(pl);
3558}
3559
3560static struct page_list *dm_integrity_alloc_page_list(unsigned n_pages)
3561{
3562 struct page_list *pl;
3563 unsigned i;
3564
3565 pl = kvmalloc_array(n_pages + 1, sizeof(struct page_list), GFP_KERNEL | __GFP_ZERO);
3566 if (!pl)
3567 return NULL;
3568
3569 for (i = 0; i < n_pages; i++) {
3570 pl[i].page = alloc_page(GFP_KERNEL);
3571 if (!pl[i].page) {
3572 dm_integrity_free_page_list(pl);
3573 return NULL;
3574 }
3575 if (i)
3576 pl[i - 1].next = &pl[i];
3577 }
3578 pl[i].page = NULL;
3579 pl[i].next = NULL;
3580
3581 return pl;
3582}
3583
3584static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl)
3585{
3586 unsigned i;
3587 for (i = 0; i < ic->journal_sections; i++)
3588 kvfree(sl[i]);
3589 kvfree(sl);
3590}
3591
3592static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic,
3593 struct page_list *pl)
3594{
3595 struct scatterlist **sl;
3596 unsigned i;
3597
3598 sl = kvmalloc_array(ic->journal_sections,
3599 sizeof(struct scatterlist *),
3600 GFP_KERNEL | __GFP_ZERO);
3601 if (!sl)
3602 return NULL;
3603
3604 for (i = 0; i < ic->journal_sections; i++) {
3605 struct scatterlist *s;
3606 unsigned start_index, start_offset;
3607 unsigned end_index, end_offset;
3608 unsigned n_pages;
3609 unsigned idx;
3610
3611 page_list_location(ic, i, 0, &start_index, &start_offset);
3612 page_list_location(ic, i, ic->journal_section_sectors - 1,
3613 &end_index, &end_offset);
3614
3615 n_pages = (end_index - start_index + 1);
3616
3617 s = kvmalloc_array(n_pages, sizeof(struct scatterlist),
3618 GFP_KERNEL);
3619 if (!s) {
3620 dm_integrity_free_journal_scatterlist(ic, sl);
3621 return NULL;
3622 }
3623
3624 sg_init_table(s, n_pages);
3625 for (idx = start_index; idx <= end_index; idx++) {
3626 char *va = lowmem_page_address(pl[idx].page);
3627 unsigned start = 0, end = PAGE_SIZE;
3628 if (idx == start_index)
3629 start = start_offset;
3630 if (idx == end_index)
3631 end = end_offset + (1 << SECTOR_SHIFT);
3632 sg_set_buf(&s[idx - start_index], va + start, end - start);
3633 }
3634
3635 sl[i] = s;
3636 }
3637
3638 return sl;
3639}
3640
3641static void free_alg(struct alg_spec *a)
3642{
3643 kfree_sensitive(a->alg_string);
3644 kfree_sensitive(a->key);
3645 memset(a, 0, sizeof *a);
3646}
3647
3648static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval)
3649{
3650 char *k;
3651
3652 free_alg(a);
3653
3654 a->alg_string = kstrdup(strchr(arg, ':') + 1, GFP_KERNEL);
3655 if (!a->alg_string)
3656 goto nomem;
3657
3658 k = strchr(a->alg_string, ':');
3659 if (k) {
3660 *k = 0;
3661 a->key_string = k + 1;
3662 if (strlen(a->key_string) & 1)
3663 goto inval;
3664
3665 a->key_size = strlen(a->key_string) / 2;
3666 a->key = kmalloc(a->key_size, GFP_KERNEL);
3667 if (!a->key)
3668 goto nomem;
3669 if (hex2bin(a->key, a->key_string, a->key_size))
3670 goto inval;
3671 }
3672
3673 return 0;
3674inval:
3675 *error = error_inval;
3676 return -EINVAL;
3677nomem:
3678 *error = "Out of memory for an argument";
3679 return -ENOMEM;
3680}
3681
3682static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error,
3683 char *error_alg, char *error_key)
3684{
3685 int r;
3686
3687 if (a->alg_string) {
3688 *hash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
3689 if (IS_ERR(*hash)) {
3690 *error = error_alg;
3691 r = PTR_ERR(*hash);
3692 *hash = NULL;
3693 return r;
3694 }
3695
3696 if (a->key) {
3697 r = crypto_shash_setkey(*hash, a->key, a->key_size);
3698 if (r) {
3699 *error = error_key;
3700 return r;
3701 }
3702 } else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) {
3703 *error = error_key;
3704 return -ENOKEY;
3705 }
3706 }
3707
3708 return 0;
3709}
3710
3711static int create_journal(struct dm_integrity_c *ic, char **error)
3712{
3713 int r = 0;
3714 unsigned i;
3715 __u64 journal_pages, journal_desc_size, journal_tree_size;
3716 unsigned char *crypt_data = NULL, *crypt_iv = NULL;
3717 struct skcipher_request *req = NULL;
3718
3719 ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL);
3720 ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL);
3721 ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL);
3722 ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL);
3723
3724 journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors,
3725 PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT);
3726 journal_desc_size = journal_pages * sizeof(struct page_list);
3727 if (journal_pages >= totalram_pages() - totalhigh_pages() || journal_desc_size > ULONG_MAX) {
3728 *error = "Journal doesn't fit into memory";
3729 r = -ENOMEM;
3730 goto bad;
3731 }
3732 ic->journal_pages = journal_pages;
3733
3734 ic->journal = dm_integrity_alloc_page_list(ic->journal_pages);
3735 if (!ic->journal) {
3736 *error = "Could not allocate memory for journal";
3737 r = -ENOMEM;
3738 goto bad;
3739 }
3740 if (ic->journal_crypt_alg.alg_string) {
3741 unsigned ivsize, blocksize;
3742 struct journal_completion comp;
3743
3744 comp.ic = ic;
3745 ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
3746 if (IS_ERR(ic->journal_crypt)) {
3747 *error = "Invalid journal cipher";
3748 r = PTR_ERR(ic->journal_crypt);
3749 ic->journal_crypt = NULL;
3750 goto bad;
3751 }
3752 ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
3753 blocksize = crypto_skcipher_blocksize(ic->journal_crypt);
3754
3755 if (ic->journal_crypt_alg.key) {
3756 r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key,
3757 ic->journal_crypt_alg.key_size);
3758 if (r) {
3759 *error = "Error setting encryption key";
3760 goto bad;
3761 }
3762 }
3763 DEBUG_print("cipher %s, block size %u iv size %u\n",
3764 ic->journal_crypt_alg.alg_string, blocksize, ivsize);
3765
3766 ic->journal_io = dm_integrity_alloc_page_list(ic->journal_pages);
3767 if (!ic->journal_io) {
3768 *error = "Could not allocate memory for journal io";
3769 r = -ENOMEM;
3770 goto bad;
3771 }
3772
3773 if (blocksize == 1) {
3774 struct scatterlist *sg;
3775
3776 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
3777 if (!req) {
3778 *error = "Could not allocate crypt request";
3779 r = -ENOMEM;
3780 goto bad;
3781 }
3782
3783 crypt_iv = kzalloc(ivsize, GFP_KERNEL);
3784 if (!crypt_iv) {
3785 *error = "Could not allocate iv";
3786 r = -ENOMEM;
3787 goto bad;
3788 }
3789
3790 ic->journal_xor = dm_integrity_alloc_page_list(ic->journal_pages);
3791 if (!ic->journal_xor) {
3792 *error = "Could not allocate memory for journal xor";
3793 r = -ENOMEM;
3794 goto bad;
3795 }
3796
3797 sg = kvmalloc_array(ic->journal_pages + 1,
3798 sizeof(struct scatterlist),
3799 GFP_KERNEL);
3800 if (!sg) {
3801 *error = "Unable to allocate sg list";
3802 r = -ENOMEM;
3803 goto bad;
3804 }
3805 sg_init_table(sg, ic->journal_pages + 1);
3806 for (i = 0; i < ic->journal_pages; i++) {
3807 char *va = lowmem_page_address(ic->journal_xor[i].page);
3808 clear_page(va);
3809 sg_set_buf(&sg[i], va, PAGE_SIZE);
3810 }
3811 sg_set_buf(&sg[i], &ic->commit_ids, sizeof ic->commit_ids);
3812
3813 skcipher_request_set_crypt(req, sg, sg,
3814 PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, crypt_iv);
3815 init_completion(&comp.comp);
3816 comp.in_flight = (atomic_t)ATOMIC_INIT(1);
3817 if (do_crypt(true, req, &comp))
3818 wait_for_completion(&comp.comp);
3819 kvfree(sg);
3820 r = dm_integrity_failed(ic);
3821 if (r) {
3822 *error = "Unable to encrypt journal";
3823 goto bad;
3824 }
3825 DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data");
3826
3827 crypto_free_skcipher(ic->journal_crypt);
3828 ic->journal_crypt = NULL;
3829 } else {
3830 unsigned crypt_len = roundup(ivsize, blocksize);
3831
3832 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
3833 if (!req) {
3834 *error = "Could not allocate crypt request";
3835 r = -ENOMEM;
3836 goto bad;
3837 }
3838
3839 crypt_iv = kmalloc(ivsize, GFP_KERNEL);
3840 if (!crypt_iv) {
3841 *error = "Could not allocate iv";
3842 r = -ENOMEM;
3843 goto bad;
3844 }
3845
3846 crypt_data = kmalloc(crypt_len, GFP_KERNEL);
3847 if (!crypt_data) {
3848 *error = "Unable to allocate crypt data";
3849 r = -ENOMEM;
3850 goto bad;
3851 }
3852
3853 ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal);
3854 if (!ic->journal_scatterlist) {
3855 *error = "Unable to allocate sg list";
3856 r = -ENOMEM;
3857 goto bad;
3858 }
3859 ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io);
3860 if (!ic->journal_io_scatterlist) {
3861 *error = "Unable to allocate sg list";
3862 r = -ENOMEM;
3863 goto bad;
3864 }
3865 ic->sk_requests = kvmalloc_array(ic->journal_sections,
3866 sizeof(struct skcipher_request *),
3867 GFP_KERNEL | __GFP_ZERO);
3868 if (!ic->sk_requests) {
3869 *error = "Unable to allocate sk requests";
3870 r = -ENOMEM;
3871 goto bad;
3872 }
3873 for (i = 0; i < ic->journal_sections; i++) {
3874 struct scatterlist sg;
3875 struct skcipher_request *section_req;
3876 __le32 section_le = cpu_to_le32(i);
3877
3878 memset(crypt_iv, 0x00, ivsize);
3879 memset(crypt_data, 0x00, crypt_len);
3880 memcpy(crypt_data, §ion_le, min((size_t)crypt_len, sizeof(section_le)));
3881
3882 sg_init_one(&sg, crypt_data, crypt_len);
3883 skcipher_request_set_crypt(req, &sg, &sg, crypt_len, crypt_iv);
3884 init_completion(&comp.comp);
3885 comp.in_flight = (atomic_t)ATOMIC_INIT(1);
3886 if (do_crypt(true, req, &comp))
3887 wait_for_completion(&comp.comp);
3888
3889 r = dm_integrity_failed(ic);
3890 if (r) {
3891 *error = "Unable to generate iv";
3892 goto bad;
3893 }
3894
3895 section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
3896 if (!section_req) {
3897 *error = "Unable to allocate crypt request";
3898 r = -ENOMEM;
3899 goto bad;
3900 }
3901 section_req->iv = kmalloc_array(ivsize, 2,
3902 GFP_KERNEL);
3903 if (!section_req->iv) {
3904 skcipher_request_free(section_req);
3905 *error = "Unable to allocate iv";
3906 r = -ENOMEM;
3907 goto bad;
3908 }
3909 memcpy(section_req->iv + ivsize, crypt_data, ivsize);
3910 section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT;
3911 ic->sk_requests[i] = section_req;
3912 DEBUG_bytes(crypt_data, ivsize, "iv(%u)", i);
3913 }
3914 }
3915 }
3916
3917 for (i = 0; i < N_COMMIT_IDS; i++) {
3918 unsigned j;
3919retest_commit_id:
3920 for (j = 0; j < i; j++) {
3921 if (ic->commit_ids[j] == ic->commit_ids[i]) {
3922 ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1);
3923 goto retest_commit_id;
3924 }
3925 }
3926 DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]);
3927 }
3928
3929 journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node);
3930 if (journal_tree_size > ULONG_MAX) {
3931 *error = "Journal doesn't fit into memory";
3932 r = -ENOMEM;
3933 goto bad;
3934 }
3935 ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL);
3936 if (!ic->journal_tree) {
3937 *error = "Could not allocate memory for journal tree";
3938 r = -ENOMEM;
3939 }
3940bad:
3941 kfree(crypt_data);
3942 kfree(crypt_iv);
3943 skcipher_request_free(req);
3944
3945 return r;
3946}
3947
3948/*
3949 * Construct a integrity mapping
3950 *
3951 * Arguments:
3952 * device
3953 * offset from the start of the device
3954 * tag size
3955 * D - direct writes, J - journal writes, B - bitmap mode, R - recovery mode
3956 * number of optional arguments
3957 * optional arguments:
3958 * journal_sectors
3959 * interleave_sectors
3960 * buffer_sectors
3961 * journal_watermark
3962 * commit_time
3963 * meta_device
3964 * block_size
3965 * sectors_per_bit
3966 * bitmap_flush_interval
3967 * internal_hash
3968 * journal_crypt
3969 * journal_mac
3970 * recalculate
3971 */
3972static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
3973{
3974 struct dm_integrity_c *ic;
3975 char dummy;
3976 int r;
3977 unsigned extra_args;
3978 struct dm_arg_set as;
3979 static const struct dm_arg _args[] = {
3980 {0, 18, "Invalid number of feature args"},
3981 };
3982 unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
3983 bool should_write_sb;
3984 __u64 threshold;
3985 unsigned long long start;
3986 __s8 log2_sectors_per_bitmap_bit = -1;
3987 __s8 log2_blocks_per_bitmap_bit;
3988 __u64 bits_in_journal;
3989 __u64 n_bitmap_bits;
3990
3991#define DIRECT_ARGUMENTS 4
3992
3993 if (argc <= DIRECT_ARGUMENTS) {
3994 ti->error = "Invalid argument count";
3995 return -EINVAL;
3996 }
3997
3998 ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL);
3999 if (!ic) {
4000 ti->error = "Cannot allocate integrity context";
4001 return -ENOMEM;
4002 }
4003 ti->private = ic;
4004 ti->per_io_data_size = sizeof(struct dm_integrity_io);
4005 ic->ti = ti;
4006
4007 ic->in_progress = RB_ROOT;
4008 INIT_LIST_HEAD(&ic->wait_list);
4009 init_waitqueue_head(&ic->endio_wait);
4010 bio_list_init(&ic->flush_bio_list);
4011 init_waitqueue_head(&ic->copy_to_journal_wait);
4012 init_completion(&ic->crypto_backoff);
4013 atomic64_set(&ic->number_of_mismatches, 0);
4014 ic->bitmap_flush_interval = BITMAP_FLUSH_INTERVAL;
4015
4016 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev);
4017 if (r) {
4018 ti->error = "Device lookup failed";
4019 goto bad;
4020 }
4021
4022 if (sscanf(argv[1], "%llu%c", &start, &dummy) != 1 || start != (sector_t)start) {
4023 ti->error = "Invalid starting offset";
4024 r = -EINVAL;
4025 goto bad;
4026 }
4027 ic->start = start;
4028
4029 if (strcmp(argv[2], "-")) {
4030 if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) {
4031 ti->error = "Invalid tag size";
4032 r = -EINVAL;
4033 goto bad;
4034 }
4035 }
4036
4037 if (!strcmp(argv[3], "J") || !strcmp(argv[3], "B") ||
4038 !strcmp(argv[3], "D") || !strcmp(argv[3], "R")) {
4039 ic->mode = argv[3][0];
4040 } else {
4041 ti->error = "Invalid mode (expecting J, B, D, R)";
4042 r = -EINVAL;
4043 goto bad;
4044 }
4045
4046 journal_sectors = 0;
4047 interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
4048 buffer_sectors = DEFAULT_BUFFER_SECTORS;
4049 journal_watermark = DEFAULT_JOURNAL_WATERMARK;
4050 sync_msec = DEFAULT_SYNC_MSEC;
4051 ic->sectors_per_block = 1;
4052
4053 as.argc = argc - DIRECT_ARGUMENTS;
4054 as.argv = argv + DIRECT_ARGUMENTS;
4055 r = dm_read_arg_group(_args, &as, &extra_args, &ti->error);
4056 if (r)
4057 goto bad;
4058
4059 while (extra_args--) {
4060 const char *opt_string;
4061 unsigned val;
4062 unsigned long long llval;
4063 opt_string = dm_shift_arg(&as);
4064 if (!opt_string) {
4065 r = -EINVAL;
4066 ti->error = "Not enough feature arguments";
4067 goto bad;
4068 }
4069 if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1)
4070 journal_sectors = val ? val : 1;
4071 else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1)
4072 interleave_sectors = val;
4073 else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1)
4074 buffer_sectors = val;
4075 else if (sscanf(opt_string, "journal_watermark:%u%c", &val, &dummy) == 1 && val <= 100)
4076 journal_watermark = val;
4077 else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1)
4078 sync_msec = val;
4079 else if (!strncmp(opt_string, "meta_device:", strlen("meta_device:"))) {
4080 if (ic->meta_dev) {
4081 dm_put_device(ti, ic->meta_dev);
4082 ic->meta_dev = NULL;
4083 }
4084 r = dm_get_device(ti, strchr(opt_string, ':') + 1,
4085 dm_table_get_mode(ti->table), &ic->meta_dev);
4086 if (r) {
4087 ti->error = "Device lookup failed";
4088 goto bad;
4089 }
4090 } else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) {
4091 if (val < 1 << SECTOR_SHIFT ||
4092 val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT ||
4093 (val & (val -1))) {
4094 r = -EINVAL;
4095 ti->error = "Invalid block_size argument";
4096 goto bad;
4097 }
4098 ic->sectors_per_block = val >> SECTOR_SHIFT;
4099 } else if (sscanf(opt_string, "sectors_per_bit:%llu%c", &llval, &dummy) == 1) {
4100 log2_sectors_per_bitmap_bit = !llval ? 0 : __ilog2_u64(llval);
4101 } else if (sscanf(opt_string, "bitmap_flush_interval:%u%c", &val, &dummy) == 1) {
4102 if (val >= (uint64_t)UINT_MAX * 1000 / HZ) {
4103 r = -EINVAL;
4104 ti->error = "Invalid bitmap_flush_interval argument";
4105 goto bad;
4106 }
4107 ic->bitmap_flush_interval = msecs_to_jiffies(val);
4108 } else if (!strncmp(opt_string, "internal_hash:", strlen("internal_hash:"))) {
4109 r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error,
4110 "Invalid internal_hash argument");
4111 if (r)
4112 goto bad;
4113 } else if (!strncmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) {
4114 r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error,
4115 "Invalid journal_crypt argument");
4116 if (r)
4117 goto bad;
4118 } else if (!strncmp(opt_string, "journal_mac:", strlen("journal_mac:"))) {
4119 r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error,
4120 "Invalid journal_mac argument");
4121 if (r)
4122 goto bad;
4123 } else if (!strcmp(opt_string, "recalculate")) {
4124 ic->recalculate_flag = true;
4125 } else if (!strcmp(opt_string, "reset_recalculate")) {
4126 ic->recalculate_flag = true;
4127 ic->reset_recalculate_flag = true;
4128 } else if (!strcmp(opt_string, "allow_discards")) {
4129 ic->discard = true;
4130 } else if (!strcmp(opt_string, "fix_padding")) {
4131 ic->fix_padding = true;
4132 } else if (!strcmp(opt_string, "fix_hmac")) {
4133 ic->fix_hmac = true;
4134 } else if (!strcmp(opt_string, "legacy_recalculate")) {
4135 ic->legacy_recalculate = true;
4136 } else {
4137 r = -EINVAL;
4138 ti->error = "Invalid argument";
4139 goto bad;
4140 }
4141 }
4142
4143 ic->data_device_sectors = bdev_nr_sectors(ic->dev->bdev);
4144 if (!ic->meta_dev)
4145 ic->meta_device_sectors = ic->data_device_sectors;
4146 else
4147 ic->meta_device_sectors = bdev_nr_sectors(ic->meta_dev->bdev);
4148
4149 if (!journal_sectors) {
4150 journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS,
4151 ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR);
4152 }
4153
4154 if (!buffer_sectors)
4155 buffer_sectors = 1;
4156 ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT);
4157
4158 r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error,
4159 "Invalid internal hash", "Error setting internal hash key");
4160 if (r)
4161 goto bad;
4162
4163 r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error,
4164 "Invalid journal mac", "Error setting journal mac key");
4165 if (r)
4166 goto bad;
4167
4168 if (!ic->tag_size) {
4169 if (!ic->internal_hash) {
4170 ti->error = "Unknown tag size";
4171 r = -EINVAL;
4172 goto bad;
4173 }
4174 ic->tag_size = crypto_shash_digestsize(ic->internal_hash);
4175 }
4176 if (ic->tag_size > MAX_TAG_SIZE) {
4177 ti->error = "Too big tag size";
4178 r = -EINVAL;
4179 goto bad;
4180 }
4181 if (!(ic->tag_size & (ic->tag_size - 1)))
4182 ic->log2_tag_size = __ffs(ic->tag_size);
4183 else
4184 ic->log2_tag_size = -1;
4185
4186 if (ic->mode == 'B' && !ic->internal_hash) {
4187 r = -EINVAL;
4188 ti->error = "Bitmap mode can be only used with internal hash";
4189 goto bad;
4190 }
4191
4192 if (ic->discard && !ic->internal_hash) {
4193 r = -EINVAL;
4194 ti->error = "Discard can be only used with internal hash";
4195 goto bad;
4196 }
4197
4198 ic->autocommit_jiffies = msecs_to_jiffies(sync_msec);
4199 ic->autocommit_msec = sync_msec;
4200 timer_setup(&ic->autocommit_timer, autocommit_fn, 0);
4201
4202 ic->io = dm_io_client_create();
4203 if (IS_ERR(ic->io)) {
4204 r = PTR_ERR(ic->io);
4205 ic->io = NULL;
4206 ti->error = "Cannot allocate dm io";
4207 goto bad;
4208 }
4209
4210 r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache);
4211 if (r) {
4212 ti->error = "Cannot allocate mempool";
4213 goto bad;
4214 }
4215
4216 ic->metadata_wq = alloc_workqueue("dm-integrity-metadata",
4217 WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE);
4218 if (!ic->metadata_wq) {
4219 ti->error = "Cannot allocate workqueue";
4220 r = -ENOMEM;
4221 goto bad;
4222 }
4223
4224 /*
4225 * If this workqueue were percpu, it would cause bio reordering
4226 * and reduced performance.
4227 */
4228 ic->wait_wq = alloc_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
4229 if (!ic->wait_wq) {
4230 ti->error = "Cannot allocate workqueue";
4231 r = -ENOMEM;
4232 goto bad;
4233 }
4234
4235 ic->offload_wq = alloc_workqueue("dm-integrity-offload", WQ_MEM_RECLAIM,
4236 METADATA_WORKQUEUE_MAX_ACTIVE);
4237 if (!ic->offload_wq) {
4238 ti->error = "Cannot allocate workqueue";
4239 r = -ENOMEM;
4240 goto bad;
4241 }
4242
4243 ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1);
4244 if (!ic->commit_wq) {
4245 ti->error = "Cannot allocate workqueue";
4246 r = -ENOMEM;
4247 goto bad;
4248 }
4249 INIT_WORK(&ic->commit_work, integrity_commit);
4250
4251 if (ic->mode == 'J' || ic->mode == 'B') {
4252 ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1);
4253 if (!ic->writer_wq) {
4254 ti->error = "Cannot allocate workqueue";
4255 r = -ENOMEM;
4256 goto bad;
4257 }
4258 INIT_WORK(&ic->writer_work, integrity_writer);
4259 }
4260
4261 ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL);
4262 if (!ic->sb) {
4263 r = -ENOMEM;
4264 ti->error = "Cannot allocate superblock area";
4265 goto bad;
4266 }
4267
4268 r = sync_rw_sb(ic, REQ_OP_READ);
4269 if (r) {
4270 ti->error = "Error reading superblock";
4271 goto bad;
4272 }
4273 should_write_sb = false;
4274 if (memcmp(ic->sb->magic, SB_MAGIC, 8)) {
4275 if (ic->mode != 'R') {
4276 if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) {
4277 r = -EINVAL;
4278 ti->error = "The device is not initialized";
4279 goto bad;
4280 }
4281 }
4282
4283 r = initialize_superblock(ic, journal_sectors, interleave_sectors);
4284 if (r) {
4285 ti->error = "Could not initialize superblock";
4286 goto bad;
4287 }
4288 if (ic->mode != 'R')
4289 should_write_sb = true;
4290 }
4291
4292 if (!ic->sb->version || ic->sb->version > SB_VERSION_5) {
4293 r = -EINVAL;
4294 ti->error = "Unknown version";
4295 goto bad;
4296 }
4297 if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) {
4298 r = -EINVAL;
4299 ti->error = "Tag size doesn't match the information in superblock";
4300 goto bad;
4301 }
4302 if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) {
4303 r = -EINVAL;
4304 ti->error = "Block size doesn't match the information in superblock";
4305 goto bad;
4306 }
4307 if (!le32_to_cpu(ic->sb->journal_sections)) {
4308 r = -EINVAL;
4309 ti->error = "Corrupted superblock, journal_sections is 0";
4310 goto bad;
4311 }
4312 /* make sure that ti->max_io_len doesn't overflow */
4313 if (!ic->meta_dev) {
4314 if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS ||
4315 ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) {
4316 r = -EINVAL;
4317 ti->error = "Invalid interleave_sectors in the superblock";
4318 goto bad;
4319 }
4320 } else {
4321 if (ic->sb->log2_interleave_sectors) {
4322 r = -EINVAL;
4323 ti->error = "Invalid interleave_sectors in the superblock";
4324 goto bad;
4325 }
4326 }
4327 if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) {
4328 r = -EINVAL;
4329 ti->error = "Journal mac mismatch";
4330 goto bad;
4331 }
4332
4333 get_provided_data_sectors(ic);
4334 if (!ic->provided_data_sectors) {
4335 r = -EINVAL;
4336 ti->error = "The device is too small";
4337 goto bad;
4338 }
4339
4340try_smaller_buffer:
4341 r = calculate_device_limits(ic);
4342 if (r) {
4343 if (ic->meta_dev) {
4344 if (ic->log2_buffer_sectors > 3) {
4345 ic->log2_buffer_sectors--;
4346 goto try_smaller_buffer;
4347 }
4348 }
4349 ti->error = "The device is too small";
4350 goto bad;
4351 }
4352
4353 if (log2_sectors_per_bitmap_bit < 0)
4354 log2_sectors_per_bitmap_bit = __fls(DEFAULT_SECTORS_PER_BITMAP_BIT);
4355 if (log2_sectors_per_bitmap_bit < ic->sb->log2_sectors_per_block)
4356 log2_sectors_per_bitmap_bit = ic->sb->log2_sectors_per_block;
4357
4358 bits_in_journal = ((__u64)ic->journal_section_sectors * ic->journal_sections) << (SECTOR_SHIFT + 3);
4359 if (bits_in_journal > UINT_MAX)
4360 bits_in_journal = UINT_MAX;
4361 while (bits_in_journal < (ic->provided_data_sectors + ((sector_t)1 << log2_sectors_per_bitmap_bit) - 1) >> log2_sectors_per_bitmap_bit)
4362 log2_sectors_per_bitmap_bit++;
4363
4364 log2_blocks_per_bitmap_bit = log2_sectors_per_bitmap_bit - ic->sb->log2_sectors_per_block;
4365 ic->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit;
4366 if (should_write_sb) {
4367 ic->sb->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit;
4368 }
4369 n_bitmap_bits = ((ic->provided_data_sectors >> ic->sb->log2_sectors_per_block)
4370 + (((sector_t)1 << log2_blocks_per_bitmap_bit) - 1)) >> log2_blocks_per_bitmap_bit;
4371 ic->n_bitmap_blocks = DIV_ROUND_UP(n_bitmap_bits, BITMAP_BLOCK_SIZE * 8);
4372
4373 if (!ic->meta_dev)
4374 ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run));
4375
4376 if (ti->len > ic->provided_data_sectors) {
4377 r = -EINVAL;
4378 ti->error = "Not enough provided sectors for requested mapping size";
4379 goto bad;
4380 }
4381
4382
4383 threshold = (__u64)ic->journal_entries * (100 - journal_watermark);
4384 threshold += 50;
4385 do_div(threshold, 100);
4386 ic->free_sectors_threshold = threshold;
4387
4388 DEBUG_print("initialized:\n");
4389 DEBUG_print(" integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size));
4390 DEBUG_print(" journal_entry_size %u\n", ic->journal_entry_size);
4391 DEBUG_print(" journal_entries_per_sector %u\n", ic->journal_entries_per_sector);
4392 DEBUG_print(" journal_section_entries %u\n", ic->journal_section_entries);
4393 DEBUG_print(" journal_section_sectors %u\n", ic->journal_section_sectors);
4394 DEBUG_print(" journal_sections %u\n", (unsigned)le32_to_cpu(ic->sb->journal_sections));
4395 DEBUG_print(" journal_entries %u\n", ic->journal_entries);
4396 DEBUG_print(" log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors);
4397 DEBUG_print(" data_device_sectors 0x%llx\n", bdev_nr_sectors(ic->dev->bdev));
4398 DEBUG_print(" initial_sectors 0x%x\n", ic->initial_sectors);
4399 DEBUG_print(" metadata_run 0x%x\n", ic->metadata_run);
4400 DEBUG_print(" log2_metadata_run %d\n", ic->log2_metadata_run);
4401 DEBUG_print(" provided_data_sectors 0x%llx (%llu)\n", ic->provided_data_sectors, ic->provided_data_sectors);
4402 DEBUG_print(" log2_buffer_sectors %u\n", ic->log2_buffer_sectors);
4403 DEBUG_print(" bits_in_journal %llu\n", bits_in_journal);
4404
4405 if (ic->recalculate_flag && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) {
4406 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
4407 ic->sb->recalc_sector = cpu_to_le64(0);
4408 }
4409
4410 if (ic->internal_hash) {
4411 size_t recalc_tags_size;
4412 ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1);
4413 if (!ic->recalc_wq ) {
4414 ti->error = "Cannot allocate workqueue";
4415 r = -ENOMEM;
4416 goto bad;
4417 }
4418 INIT_WORK(&ic->recalc_work, integrity_recalc);
4419 ic->recalc_buffer = vmalloc(RECALC_SECTORS << SECTOR_SHIFT);
4420 if (!ic->recalc_buffer) {
4421 ti->error = "Cannot allocate buffer for recalculating";
4422 r = -ENOMEM;
4423 goto bad;
4424 }
4425 recalc_tags_size = (RECALC_SECTORS >> ic->sb->log2_sectors_per_block) * ic->tag_size;
4426 if (crypto_shash_digestsize(ic->internal_hash) > ic->tag_size)
4427 recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tag_size;
4428 ic->recalc_tags = kvmalloc(recalc_tags_size, GFP_KERNEL);
4429 if (!ic->recalc_tags) {
4430 ti->error = "Cannot allocate tags for recalculating";
4431 r = -ENOMEM;
4432 goto bad;
4433 }
4434 } else {
4435 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
4436 ti->error = "Recalculate can only be specified with internal_hash";
4437 r = -EINVAL;
4438 goto bad;
4439 }
4440 }
4441
4442 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
4443 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors &&
4444 dm_integrity_disable_recalculate(ic)) {
4445 ti->error = "Recalculating with HMAC is disabled for security reasons - if you really need it, use the argument \"legacy_recalculate\"";
4446 r = -EOPNOTSUPP;
4447 goto bad;
4448 }
4449
4450 ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev,
4451 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL, 0);
4452 if (IS_ERR(ic->bufio)) {
4453 r = PTR_ERR(ic->bufio);
4454 ti->error = "Cannot initialize dm-bufio";
4455 ic->bufio = NULL;
4456 goto bad;
4457 }
4458 dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors);
4459
4460 if (ic->mode != 'R') {
4461 r = create_journal(ic, &ti->error);
4462 if (r)
4463 goto bad;
4464
4465 }
4466
4467 if (ic->mode == 'B') {
4468 unsigned i;
4469 unsigned n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE);
4470
4471 ic->recalc_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages);
4472 if (!ic->recalc_bitmap) {
4473 r = -ENOMEM;
4474 goto bad;
4475 }
4476 ic->may_write_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages);
4477 if (!ic->may_write_bitmap) {
4478 r = -ENOMEM;
4479 goto bad;
4480 }
4481 ic->bbs = kvmalloc_array(ic->n_bitmap_blocks, sizeof(struct bitmap_block_status), GFP_KERNEL);
4482 if (!ic->bbs) {
4483 r = -ENOMEM;
4484 goto bad;
4485 }
4486 INIT_DELAYED_WORK(&ic->bitmap_flush_work, bitmap_flush_work);
4487 for (i = 0; i < ic->n_bitmap_blocks; i++) {
4488 struct bitmap_block_status *bbs = &ic->bbs[i];
4489 unsigned sector, pl_index, pl_offset;
4490
4491 INIT_WORK(&bbs->work, bitmap_block_work);
4492 bbs->ic = ic;
4493 bbs->idx = i;
4494 bio_list_init(&bbs->bio_queue);
4495 spin_lock_init(&bbs->bio_queue_lock);
4496
4497 sector = i * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT);
4498 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
4499 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
4500
4501 bbs->bitmap = lowmem_page_address(ic->journal[pl_index].page) + pl_offset;
4502 }
4503 }
4504
4505 if (should_write_sb) {
4506 init_journal(ic, 0, ic->journal_sections, 0);
4507 r = dm_integrity_failed(ic);
4508 if (unlikely(r)) {
4509 ti->error = "Error initializing journal";
4510 goto bad;
4511 }
4512 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
4513 if (r) {
4514 ti->error = "Error initializing superblock";
4515 goto bad;
4516 }
4517 ic->just_formatted = true;
4518 }
4519
4520 if (!ic->meta_dev) {
4521 r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors);
4522 if (r)
4523 goto bad;
4524 }
4525 if (ic->mode == 'B') {
4526 unsigned max_io_len = ((sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit) * (BITMAP_BLOCK_SIZE * 8);
4527 if (!max_io_len)
4528 max_io_len = 1U << 31;
4529 DEBUG_print("max_io_len: old %u, new %u\n", ti->max_io_len, max_io_len);
4530 if (!ti->max_io_len || ti->max_io_len > max_io_len) {
4531 r = dm_set_target_max_io_len(ti, max_io_len);
4532 if (r)
4533 goto bad;
4534 }
4535 }
4536
4537 if (!ic->internal_hash)
4538 dm_integrity_set(ti, ic);
4539
4540 ti->num_flush_bios = 1;
4541 ti->flush_supported = true;
4542 if (ic->discard)
4543 ti->num_discard_bios = 1;
4544
4545 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);
4546 return 0;
4547
4548bad:
4549 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);
4550 dm_integrity_dtr(ti);
4551 return r;
4552}
4553
4554static void dm_integrity_dtr(struct dm_target *ti)
4555{
4556 struct dm_integrity_c *ic = ti->private;
4557
4558 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
4559 BUG_ON(!list_empty(&ic->wait_list));
4560
4561 if (ic->mode == 'B')
4562 cancel_delayed_work_sync(&ic->bitmap_flush_work);
4563 if (ic->metadata_wq)
4564 destroy_workqueue(ic->metadata_wq);
4565 if (ic->wait_wq)
4566 destroy_workqueue(ic->wait_wq);
4567 if (ic->offload_wq)
4568 destroy_workqueue(ic->offload_wq);
4569 if (ic->commit_wq)
4570 destroy_workqueue(ic->commit_wq);
4571 if (ic->writer_wq)
4572 destroy_workqueue(ic->writer_wq);
4573 if (ic->recalc_wq)
4574 destroy_workqueue(ic->recalc_wq);
4575 vfree(ic->recalc_buffer);
4576 kvfree(ic->recalc_tags);
4577 kvfree(ic->bbs);
4578 if (ic->bufio)
4579 dm_bufio_client_destroy(ic->bufio);
4580 mempool_exit(&ic->journal_io_mempool);
4581 if (ic->io)
4582 dm_io_client_destroy(ic->io);
4583 if (ic->dev)
4584 dm_put_device(ti, ic->dev);
4585 if (ic->meta_dev)
4586 dm_put_device(ti, ic->meta_dev);
4587 dm_integrity_free_page_list(ic->journal);
4588 dm_integrity_free_page_list(ic->journal_io);
4589 dm_integrity_free_page_list(ic->journal_xor);
4590 dm_integrity_free_page_list(ic->recalc_bitmap);
4591 dm_integrity_free_page_list(ic->may_write_bitmap);
4592 if (ic->journal_scatterlist)
4593 dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist);
4594 if (ic->journal_io_scatterlist)
4595 dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist);
4596 if (ic->sk_requests) {
4597 unsigned i;
4598
4599 for (i = 0; i < ic->journal_sections; i++) {
4600 struct skcipher_request *req = ic->sk_requests[i];
4601 if (req) {
4602 kfree_sensitive(req->iv);
4603 skcipher_request_free(req);
4604 }
4605 }
4606 kvfree(ic->sk_requests);
4607 }
4608 kvfree(ic->journal_tree);
4609 if (ic->sb)
4610 free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT);
4611
4612 if (ic->internal_hash)
4613 crypto_free_shash(ic->internal_hash);
4614 free_alg(&ic->internal_hash_alg);
4615
4616 if (ic->journal_crypt)
4617 crypto_free_skcipher(ic->journal_crypt);
4618 free_alg(&ic->journal_crypt_alg);
4619
4620 if (ic->journal_mac)
4621 crypto_free_shash(ic->journal_mac);
4622 free_alg(&ic->journal_mac_alg);
4623
4624 kfree(ic);
4625 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);
4626}
4627
4628static struct target_type integrity_target = {
4629 .name = "integrity",
4630 .version = {1, 10, 0},
4631 .module = THIS_MODULE,
4632 .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
4633 .ctr = dm_integrity_ctr,
4634 .dtr = dm_integrity_dtr,
4635 .map = dm_integrity_map,
4636 .postsuspend = dm_integrity_postsuspend,
4637 .resume = dm_integrity_resume,
4638 .status = dm_integrity_status,
4639 .iterate_devices = dm_integrity_iterate_devices,
4640 .io_hints = dm_integrity_io_hints,
4641};
4642
4643static int __init dm_integrity_init(void)
4644{
4645 int r;
4646
4647 journal_io_cache = kmem_cache_create("integrity_journal_io",
4648 sizeof(struct journal_io), 0, 0, NULL);
4649 if (!journal_io_cache) {
4650 DMERR("can't allocate journal io cache");
4651 return -ENOMEM;
4652 }
4653
4654 r = dm_register_target(&integrity_target);
4655
4656 if (r < 0)
4657 DMERR("register failed %d", r);
4658
4659 return r;
4660}
4661
4662static void __exit dm_integrity_exit(void)
4663{
4664 dm_unregister_target(&integrity_target);
4665 kmem_cache_destroy(journal_io_cache);
4666}
4667
4668module_init(dm_integrity_init);
4669module_exit(dm_integrity_exit);
4670
4671MODULE_AUTHOR("Milan Broz");
4672MODULE_AUTHOR("Mikulas Patocka");
4673MODULE_DESCRIPTION(DM_NAME " target for integrity tags extension");
4674MODULE_LICENSE("GPL");