Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/verity/verify.c: data verification functions, i.e. hooks for ->readpages()
4 *
5 * Copyright 2019 Google LLC
6 */
7
8#include "fsverity_private.h"
9
10#include <crypto/hash.h>
11#include <linux/bio.h>
12#include <linux/ratelimit.h>
13
14static struct workqueue_struct *fsverity_read_workqueue;
15
16/**
17 * hash_at_level() - compute the location of the block's hash at the given level
18 *
19 * @params: (in) the Merkle tree parameters
20 * @dindex: (in) the index of the data block being verified
21 * @level: (in) the level of hash we want (0 is leaf level)
22 * @hindex: (out) the index of the hash block containing the wanted hash
23 * @hoffset: (out) the byte offset to the wanted hash within the hash block
24 */
25static void hash_at_level(const struct merkle_tree_params *params,
26 pgoff_t dindex, unsigned int level, pgoff_t *hindex,
27 unsigned int *hoffset)
28{
29 pgoff_t position;
30
31 /* Offset of the hash within the level's region, in hashes */
32 position = dindex >> (level * params->log_arity);
33
34 /* Index of the hash block in the tree overall */
35 *hindex = params->level_start[level] + (position >> params->log_arity);
36
37 /* Offset of the wanted hash (in bytes) within the hash block */
38 *hoffset = (position & ((1 << params->log_arity) - 1)) <<
39 (params->log_blocksize - params->log_arity);
40}
41
42/* Extract a hash from a hash page */
43static void extract_hash(struct page *hpage, unsigned int hoffset,
44 unsigned int hsize, u8 *out)
45{
46 void *virt = kmap_atomic(hpage);
47
48 memcpy(out, virt + hoffset, hsize);
49 kunmap_atomic(virt);
50}
51
52static inline int cmp_hashes(const struct fsverity_info *vi,
53 const u8 *want_hash, const u8 *real_hash,
54 pgoff_t index, int level)
55{
56 const unsigned int hsize = vi->tree_params.digest_size;
57
58 if (memcmp(want_hash, real_hash, hsize) == 0)
59 return 0;
60
61 fsverity_err(vi->inode,
62 "FILE CORRUPTED! index=%lu, level=%d, want_hash=%s:%*phN, real_hash=%s:%*phN",
63 index, level,
64 vi->tree_params.hash_alg->name, hsize, want_hash,
65 vi->tree_params.hash_alg->name, hsize, real_hash);
66 return -EBADMSG;
67}
68
69/*
70 * Verify a single data page against the file's Merkle tree.
71 *
72 * In principle, we need to verify the entire path to the root node. However,
73 * for efficiency the filesystem may cache the hash pages. Therefore we need
74 * only ascend the tree until an already-verified page is seen, as indicated by
75 * the PageChecked bit being set; then verify the path to that page.
76 *
77 * This code currently only supports the case where the verity block size is
78 * equal to PAGE_SIZE. Doing otherwise would be possible but tricky, since we
79 * wouldn't be able to use the PageChecked bit.
80 *
81 * Note that multiple processes may race to verify a hash page and mark it
82 * Checked, but it doesn't matter; the result will be the same either way.
83 *
84 * Return: true if the page is valid, else false.
85 */
86static bool verify_page(struct inode *inode, const struct fsverity_info *vi,
87 struct ahash_request *req, struct page *data_page)
88{
89 const struct merkle_tree_params *params = &vi->tree_params;
90 const unsigned int hsize = params->digest_size;
91 const pgoff_t index = data_page->index;
92 int level;
93 u8 _want_hash[FS_VERITY_MAX_DIGEST_SIZE];
94 const u8 *want_hash;
95 u8 real_hash[FS_VERITY_MAX_DIGEST_SIZE];
96 struct page *hpages[FS_VERITY_MAX_LEVELS];
97 unsigned int hoffsets[FS_VERITY_MAX_LEVELS];
98 int err;
99
100 if (WARN_ON_ONCE(!PageLocked(data_page) || PageUptodate(data_page)))
101 return false;
102
103 pr_debug_ratelimited("Verifying data page %lu...\n", index);
104
105 /*
106 * Starting at the leaf level, ascend the tree saving hash pages along
107 * the way until we find a verified hash page, indicated by PageChecked;
108 * or until we reach the root.
109 */
110 for (level = 0; level < params->num_levels; level++) {
111 pgoff_t hindex;
112 unsigned int hoffset;
113 struct page *hpage;
114
115 hash_at_level(params, index, level, &hindex, &hoffset);
116
117 pr_debug_ratelimited("Level %d: hindex=%lu, hoffset=%u\n",
118 level, hindex, hoffset);
119
120 hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode,
121 hindex);
122 if (IS_ERR(hpage)) {
123 err = PTR_ERR(hpage);
124 fsverity_err(inode,
125 "Error %d reading Merkle tree page %lu",
126 err, hindex);
127 goto out;
128 }
129
130 if (PageChecked(hpage)) {
131 extract_hash(hpage, hoffset, hsize, _want_hash);
132 want_hash = _want_hash;
133 put_page(hpage);
134 pr_debug_ratelimited("Hash page already checked, want %s:%*phN\n",
135 params->hash_alg->name,
136 hsize, want_hash);
137 goto descend;
138 }
139 pr_debug_ratelimited("Hash page not yet checked\n");
140 hpages[level] = hpage;
141 hoffsets[level] = hoffset;
142 }
143
144 want_hash = vi->root_hash;
145 pr_debug("Want root hash: %s:%*phN\n",
146 params->hash_alg->name, hsize, want_hash);
147descend:
148 /* Descend the tree verifying hash pages */
149 for (; level > 0; level--) {
150 struct page *hpage = hpages[level - 1];
151 unsigned int hoffset = hoffsets[level - 1];
152
153 err = fsverity_hash_page(params, inode, req, hpage, real_hash);
154 if (err)
155 goto out;
156 err = cmp_hashes(vi, want_hash, real_hash, index, level - 1);
157 if (err)
158 goto out;
159 SetPageChecked(hpage);
160 extract_hash(hpage, hoffset, hsize, _want_hash);
161 want_hash = _want_hash;
162 put_page(hpage);
163 pr_debug("Verified hash page at level %d, now want %s:%*phN\n",
164 level - 1, params->hash_alg->name, hsize, want_hash);
165 }
166
167 /* Finally, verify the data page */
168 err = fsverity_hash_page(params, inode, req, data_page, real_hash);
169 if (err)
170 goto out;
171 err = cmp_hashes(vi, want_hash, real_hash, index, -1);
172out:
173 for (; level > 0; level--)
174 put_page(hpages[level - 1]);
175
176 return err == 0;
177}
178
179/**
180 * fsverity_verify_page() - verify a data page
181 *
182 * Verify a page that has just been read from a verity file. The page must be a
183 * pagecache page that is still locked and not yet uptodate.
184 *
185 * Return: true if the page is valid, else false.
186 */
187bool fsverity_verify_page(struct page *page)
188{
189 struct inode *inode = page->mapping->host;
190 const struct fsverity_info *vi = inode->i_verity_info;
191 struct ahash_request *req;
192 bool valid;
193
194 req = ahash_request_alloc(vi->tree_params.hash_alg->tfm, GFP_NOFS);
195 if (unlikely(!req))
196 return false;
197
198 valid = verify_page(inode, vi, req, page);
199
200 ahash_request_free(req);
201
202 return valid;
203}
204EXPORT_SYMBOL_GPL(fsverity_verify_page);
205
206#ifdef CONFIG_BLOCK
207/**
208 * fsverity_verify_bio() - verify a 'read' bio that has just completed
209 *
210 * Verify a set of pages that have just been read from a verity file. The pages
211 * must be pagecache pages that are still locked and not yet uptodate. Pages
212 * that fail verification are set to the Error state. Verification is skipped
213 * for pages already in the Error state, e.g. due to fscrypt decryption failure.
214 *
215 * This is a helper function for use by the ->readpages() method of filesystems
216 * that issue bios to read data directly into the page cache. Filesystems that
217 * populate the page cache without issuing bios (e.g. non block-based
218 * filesystems) must instead call fsverity_verify_page() directly on each page.
219 * All filesystems must also call fsverity_verify_page() on holes.
220 */
221void fsverity_verify_bio(struct bio *bio)
222{
223 struct inode *inode = bio_first_page_all(bio)->mapping->host;
224 const struct fsverity_info *vi = inode->i_verity_info;
225 struct ahash_request *req;
226 struct bio_vec *bv;
227 struct bvec_iter_all iter_all;
228
229 req = ahash_request_alloc(vi->tree_params.hash_alg->tfm, GFP_NOFS);
230 if (unlikely(!req)) {
231 bio_for_each_segment_all(bv, bio, iter_all)
232 SetPageError(bv->bv_page);
233 return;
234 }
235
236 bio_for_each_segment_all(bv, bio, iter_all) {
237 struct page *page = bv->bv_page;
238
239 if (!PageError(page) && !verify_page(inode, vi, req, page))
240 SetPageError(page);
241 }
242
243 ahash_request_free(req);
244}
245EXPORT_SYMBOL_GPL(fsverity_verify_bio);
246#endif /* CONFIG_BLOCK */
247
248/**
249 * fsverity_enqueue_verify_work() - enqueue work on the fs-verity workqueue
250 *
251 * Enqueue verification work for asynchronous processing.
252 */
253void fsverity_enqueue_verify_work(struct work_struct *work)
254{
255 queue_work(fsverity_read_workqueue, work);
256}
257EXPORT_SYMBOL_GPL(fsverity_enqueue_verify_work);
258
259int __init fsverity_init_workqueue(void)
260{
261 /*
262 * Use an unbound workqueue to allow bios to be verified in parallel
263 * even when they happen to complete on the same CPU. This sacrifices
264 * locality, but it's worthwhile since hashing is CPU-intensive.
265 *
266 * Also use a high-priority workqueue to prioritize verification work,
267 * which blocks reads from completing, over regular application tasks.
268 */
269 fsverity_read_workqueue = alloc_workqueue("fsverity_read_queue",
270 WQ_UNBOUND | WQ_HIGHPRI,
271 num_online_cpus());
272 if (!fsverity_read_workqueue)
273 return -ENOMEM;
274 return 0;
275}
276
277void __init fsverity_exit_workqueue(void)
278{
279 destroy_workqueue(fsverity_read_workqueue);
280 fsverity_read_workqueue = NULL;
281}