Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2019 Google LLC
4 */
5
6/**
7 * DOC: The Keyslot Manager
8 *
9 * Many devices with inline encryption support have a limited number of "slots"
10 * into which encryption contexts may be programmed, and requests can be tagged
11 * with a slot number to specify the key to use for en/decryption.
12 *
13 * As the number of slots is limited, and programming keys is expensive on
14 * many inline encryption hardware, we don't want to program the same key into
15 * multiple slots - if multiple requests are using the same key, we want to
16 * program just one slot with that key and use that slot for all requests.
17 *
18 * The keyslot manager manages these keyslots appropriately, and also acts as
19 * an abstraction between the inline encryption hardware and the upper layers.
20 *
21 * Lower layer devices will set up a keyslot manager in their request queue
22 * and tell it how to perform device specific operations like programming/
23 * evicting keys from keyslots.
24 *
25 * Upper layers will call blk_ksm_get_slot_for_key() to program a
26 * key into some slot in the inline encryption hardware.
27 */
28
29#define pr_fmt(fmt) "blk-crypto: " fmt
30
31#include <linux/keyslot-manager.h>
32#include <linux/atomic.h>
33#include <linux/mutex.h>
34#include <linux/pm_runtime.h>
35#include <linux/wait.h>
36#include <linux/blkdev.h>
37
38struct blk_ksm_keyslot {
39 atomic_t slot_refs;
40 struct list_head idle_slot_node;
41 struct hlist_node hash_node;
42 const struct blk_crypto_key *key;
43 struct blk_keyslot_manager *ksm;
44};
45
46static inline void blk_ksm_hw_enter(struct blk_keyslot_manager *ksm)
47{
48 /*
49 * Calling into the driver requires ksm->lock held and the device
50 * resumed. But we must resume the device first, since that can acquire
51 * and release ksm->lock via blk_ksm_reprogram_all_keys().
52 */
53 if (ksm->dev)
54 pm_runtime_get_sync(ksm->dev);
55 down_write(&ksm->lock);
56}
57
58static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm)
59{
60 up_write(&ksm->lock);
61 if (ksm->dev)
62 pm_runtime_put_sync(ksm->dev);
63}
64
65/**
66 * blk_ksm_init() - Initialize a keyslot manager
67 * @ksm: The keyslot_manager to initialize.
68 * @num_slots: The number of key slots to manage.
69 *
70 * Allocate memory for keyslots and initialize a keyslot manager. Called by
71 * e.g. storage drivers to set up a keyslot manager in their request_queue.
72 *
73 * Return: 0 on success, or else a negative error code.
74 */
75int blk_ksm_init(struct blk_keyslot_manager *ksm, unsigned int num_slots)
76{
77 unsigned int slot;
78 unsigned int i;
79 unsigned int slot_hashtable_size;
80
81 memset(ksm, 0, sizeof(*ksm));
82
83 if (num_slots == 0)
84 return -EINVAL;
85
86 ksm->slots = kvcalloc(num_slots, sizeof(ksm->slots[0]), GFP_KERNEL);
87 if (!ksm->slots)
88 return -ENOMEM;
89
90 ksm->num_slots = num_slots;
91
92 init_rwsem(&ksm->lock);
93
94 init_waitqueue_head(&ksm->idle_slots_wait_queue);
95 INIT_LIST_HEAD(&ksm->idle_slots);
96
97 for (slot = 0; slot < num_slots; slot++) {
98 ksm->slots[slot].ksm = ksm;
99 list_add_tail(&ksm->slots[slot].idle_slot_node,
100 &ksm->idle_slots);
101 }
102
103 spin_lock_init(&ksm->idle_slots_lock);
104
105 slot_hashtable_size = roundup_pow_of_two(num_slots);
106 ksm->log_slot_ht_size = ilog2(slot_hashtable_size);
107 ksm->slot_hashtable = kvmalloc_array(slot_hashtable_size,
108 sizeof(ksm->slot_hashtable[0]),
109 GFP_KERNEL);
110 if (!ksm->slot_hashtable)
111 goto err_destroy_ksm;
112 for (i = 0; i < slot_hashtable_size; i++)
113 INIT_HLIST_HEAD(&ksm->slot_hashtable[i]);
114
115 return 0;
116
117err_destroy_ksm:
118 blk_ksm_destroy(ksm);
119 return -ENOMEM;
120}
121EXPORT_SYMBOL_GPL(blk_ksm_init);
122
123static inline struct hlist_head *
124blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm,
125 const struct blk_crypto_key *key)
126{
127 return &ksm->slot_hashtable[hash_ptr(key, ksm->log_slot_ht_size)];
128}
129
130static void blk_ksm_remove_slot_from_lru_list(struct blk_ksm_keyslot *slot)
131{
132 struct blk_keyslot_manager *ksm = slot->ksm;
133 unsigned long flags;
134
135 spin_lock_irqsave(&ksm->idle_slots_lock, flags);
136 list_del(&slot->idle_slot_node);
137 spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);
138}
139
140static struct blk_ksm_keyslot *blk_ksm_find_keyslot(
141 struct blk_keyslot_manager *ksm,
142 const struct blk_crypto_key *key)
143{
144 const struct hlist_head *head = blk_ksm_hash_bucket_for_key(ksm, key);
145 struct blk_ksm_keyslot *slotp;
146
147 hlist_for_each_entry(slotp, head, hash_node) {
148 if (slotp->key == key)
149 return slotp;
150 }
151 return NULL;
152}
153
154static struct blk_ksm_keyslot *blk_ksm_find_and_grab_keyslot(
155 struct blk_keyslot_manager *ksm,
156 const struct blk_crypto_key *key)
157{
158 struct blk_ksm_keyslot *slot;
159
160 slot = blk_ksm_find_keyslot(ksm, key);
161 if (!slot)
162 return NULL;
163 if (atomic_inc_return(&slot->slot_refs) == 1) {
164 /* Took first reference to this slot; remove it from LRU list */
165 blk_ksm_remove_slot_from_lru_list(slot);
166 }
167 return slot;
168}
169
170unsigned int blk_ksm_get_slot_idx(struct blk_ksm_keyslot *slot)
171{
172 return slot - slot->ksm->slots;
173}
174EXPORT_SYMBOL_GPL(blk_ksm_get_slot_idx);
175
176/**
177 * blk_ksm_get_slot_for_key() - Program a key into a keyslot.
178 * @ksm: The keyslot manager to program the key into.
179 * @key: Pointer to the key object to program, including the raw key, crypto
180 * mode, and data unit size.
181 * @slot_ptr: A pointer to return the pointer of the allocated keyslot.
182 *
183 * Get a keyslot that's been programmed with the specified key. If one already
184 * exists, return it with incremented refcount. Otherwise, wait for a keyslot
185 * to become idle and program it.
186 *
187 * Context: Process context. Takes and releases ksm->lock.
188 * Return: BLK_STS_OK on success (and keyslot is set to the pointer of the
189 * allocated keyslot), or some other blk_status_t otherwise (and
190 * keyslot is set to NULL).
191 */
192blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm,
193 const struct blk_crypto_key *key,
194 struct blk_ksm_keyslot **slot_ptr)
195{
196 struct blk_ksm_keyslot *slot;
197 int slot_idx;
198 int err;
199
200 *slot_ptr = NULL;
201 down_read(&ksm->lock);
202 slot = blk_ksm_find_and_grab_keyslot(ksm, key);
203 up_read(&ksm->lock);
204 if (slot)
205 goto success;
206
207 for (;;) {
208 blk_ksm_hw_enter(ksm);
209 slot = blk_ksm_find_and_grab_keyslot(ksm, key);
210 if (slot) {
211 blk_ksm_hw_exit(ksm);
212 goto success;
213 }
214
215 /*
216 * If we're here, that means there wasn't a slot that was
217 * already programmed with the key. So try to program it.
218 */
219 if (!list_empty(&ksm->idle_slots))
220 break;
221
222 blk_ksm_hw_exit(ksm);
223 wait_event(ksm->idle_slots_wait_queue,
224 !list_empty(&ksm->idle_slots));
225 }
226
227 slot = list_first_entry(&ksm->idle_slots, struct blk_ksm_keyslot,
228 idle_slot_node);
229 slot_idx = blk_ksm_get_slot_idx(slot);
230
231 err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot_idx);
232 if (err) {
233 wake_up(&ksm->idle_slots_wait_queue);
234 blk_ksm_hw_exit(ksm);
235 return errno_to_blk_status(err);
236 }
237
238 /* Move this slot to the hash list for the new key. */
239 if (slot->key)
240 hlist_del(&slot->hash_node);
241 slot->key = key;
242 hlist_add_head(&slot->hash_node, blk_ksm_hash_bucket_for_key(ksm, key));
243
244 atomic_set(&slot->slot_refs, 1);
245
246 blk_ksm_remove_slot_from_lru_list(slot);
247
248 blk_ksm_hw_exit(ksm);
249success:
250 *slot_ptr = slot;
251 return BLK_STS_OK;
252}
253
254/**
255 * blk_ksm_put_slot() - Release a reference to a slot
256 * @slot: The keyslot to release the reference of.
257 *
258 * Context: Any context.
259 */
260void blk_ksm_put_slot(struct blk_ksm_keyslot *slot)
261{
262 struct blk_keyslot_manager *ksm;
263 unsigned long flags;
264
265 if (!slot)
266 return;
267
268 ksm = slot->ksm;
269
270 if (atomic_dec_and_lock_irqsave(&slot->slot_refs,
271 &ksm->idle_slots_lock, flags)) {
272 list_add_tail(&slot->idle_slot_node, &ksm->idle_slots);
273 spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);
274 wake_up(&ksm->idle_slots_wait_queue);
275 }
276}
277
278/**
279 * blk_ksm_crypto_cfg_supported() - Find out if a crypto configuration is
280 * supported by a ksm.
281 * @ksm: The keyslot manager to check
282 * @cfg: The crypto configuration to check for.
283 *
284 * Checks for crypto_mode/data unit size/dun bytes support.
285 *
286 * Return: Whether or not this ksm supports the specified crypto config.
287 */
288bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager *ksm,
289 const struct blk_crypto_config *cfg)
290{
291 if (!ksm)
292 return false;
293 if (!(ksm->crypto_modes_supported[cfg->crypto_mode] &
294 cfg->data_unit_size))
295 return false;
296 if (ksm->max_dun_bytes_supported < cfg->dun_bytes)
297 return false;
298 return true;
299}
300
301/**
302 * blk_ksm_evict_key() - Evict a key from the lower layer device.
303 * @ksm: The keyslot manager to evict from
304 * @key: The key to evict
305 *
306 * Find the keyslot that the specified key was programmed into, and evict that
307 * slot from the lower layer device. The slot must not be in use by any
308 * in-flight IO when this function is called.
309 *
310 * Context: Process context. Takes and releases ksm->lock.
311 * Return: 0 on success or if there's no keyslot with the specified key, -EBUSY
312 * if the keyslot is still in use, or another -errno value on other
313 * error.
314 */
315int blk_ksm_evict_key(struct blk_keyslot_manager *ksm,
316 const struct blk_crypto_key *key)
317{
318 struct blk_ksm_keyslot *slot;
319 int err = 0;
320
321 blk_ksm_hw_enter(ksm);
322 slot = blk_ksm_find_keyslot(ksm, key);
323 if (!slot)
324 goto out_unlock;
325
326 if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) {
327 err = -EBUSY;
328 goto out_unlock;
329 }
330 err = ksm->ksm_ll_ops.keyslot_evict(ksm, key,
331 blk_ksm_get_slot_idx(slot));
332 if (err)
333 goto out_unlock;
334
335 hlist_del(&slot->hash_node);
336 slot->key = NULL;
337 err = 0;
338out_unlock:
339 blk_ksm_hw_exit(ksm);
340 return err;
341}
342
343/**
344 * blk_ksm_reprogram_all_keys() - Re-program all keyslots.
345 * @ksm: The keyslot manager
346 *
347 * Re-program all keyslots that are supposed to have a key programmed. This is
348 * intended only for use by drivers for hardware that loses its keys on reset.
349 *
350 * Context: Process context. Takes and releases ksm->lock.
351 */
352void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm)
353{
354 unsigned int slot;
355
356 /* This is for device initialization, so don't resume the device */
357 down_write(&ksm->lock);
358 for (slot = 0; slot < ksm->num_slots; slot++) {
359 const struct blk_crypto_key *key = ksm->slots[slot].key;
360 int err;
361
362 if (!key)
363 continue;
364
365 err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot);
366 WARN_ON(err);
367 }
368 up_write(&ksm->lock);
369}
370EXPORT_SYMBOL_GPL(blk_ksm_reprogram_all_keys);
371
372void blk_ksm_destroy(struct blk_keyslot_manager *ksm)
373{
374 if (!ksm)
375 return;
376 kvfree(ksm->slot_hashtable);
377 kvfree_sensitive(ksm->slots, sizeof(ksm->slots[0]) * ksm->num_slots);
378 memzero_explicit(ksm, sizeof(*ksm));
379}
380EXPORT_SYMBOL_GPL(blk_ksm_destroy);
381
382bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q)
383{
384 if (blk_integrity_queue_supports_integrity(q)) {
385 pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n");
386 return false;
387 }
388 q->ksm = ksm;
389 return true;
390}
391EXPORT_SYMBOL_GPL(blk_ksm_register);
392
393void blk_ksm_unregister(struct request_queue *q)
394{
395 q->ksm = NULL;
396}