Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * CXL Flash Device Driver
4 *
5 * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation
6 * Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
7 *
8 * Copyright (C) 2015 IBM Corporation
9 */
10
11#include <linux/interrupt.h>
12#include <linux/pci.h>
13#include <linux/syscalls.h>
14#include <asm/unaligned.h>
15#include <asm/bitsperlong.h>
16
17#include <scsi/scsi_cmnd.h>
18#include <scsi/scsi_host.h>
19#include <uapi/scsi/cxlflash_ioctl.h>
20
21#include "sislite.h"
22#include "common.h"
23#include "vlun.h"
24#include "superpipe.h"
25
26/**
27 * marshal_virt_to_resize() - translate uvirtual to resize structure
28 * @virt: Source structure from which to translate/copy.
29 * @resize: Destination structure for the translate/copy.
30 */
31static void marshal_virt_to_resize(struct dk_cxlflash_uvirtual *virt,
32 struct dk_cxlflash_resize *resize)
33{
34 resize->hdr = virt->hdr;
35 resize->context_id = virt->context_id;
36 resize->rsrc_handle = virt->rsrc_handle;
37 resize->req_size = virt->lun_size;
38 resize->last_lba = virt->last_lba;
39}
40
41/**
42 * marshal_clone_to_rele() - translate clone to release structure
43 * @clone: Source structure from which to translate/copy.
44 * @release: Destination structure for the translate/copy.
45 */
46static void marshal_clone_to_rele(struct dk_cxlflash_clone *clone,
47 struct dk_cxlflash_release *release)
48{
49 release->hdr = clone->hdr;
50 release->context_id = clone->context_id_dst;
51}
52
53/**
54 * ba_init() - initializes a block allocator
55 * @ba_lun: Block allocator to initialize.
56 *
57 * Return: 0 on success, -errno on failure
58 */
59static int ba_init(struct ba_lun *ba_lun)
60{
61 struct ba_lun_info *bali = NULL;
62 int lun_size_au = 0, i = 0;
63 int last_word_underflow = 0;
64 u64 *lam;
65
66 pr_debug("%s: Initializing LUN: lun_id=%016llx "
67 "ba_lun->lsize=%lx ba_lun->au_size=%lX\n",
68 __func__, ba_lun->lun_id, ba_lun->lsize, ba_lun->au_size);
69
70 /* Calculate bit map size */
71 lun_size_au = ba_lun->lsize / ba_lun->au_size;
72 if (lun_size_au == 0) {
73 pr_debug("%s: Requested LUN size of 0!\n", __func__);
74 return -EINVAL;
75 }
76
77 /* Allocate lun information container */
78 bali = kzalloc(sizeof(struct ba_lun_info), GFP_KERNEL);
79 if (unlikely(!bali)) {
80 pr_err("%s: Failed to allocate lun_info lun_id=%016llx\n",
81 __func__, ba_lun->lun_id);
82 return -ENOMEM;
83 }
84
85 bali->total_aus = lun_size_au;
86 bali->lun_bmap_size = lun_size_au / BITS_PER_LONG;
87
88 if (lun_size_au % BITS_PER_LONG)
89 bali->lun_bmap_size++;
90
91 /* Allocate bitmap space */
92 bali->lun_alloc_map = kzalloc((bali->lun_bmap_size * sizeof(u64)),
93 GFP_KERNEL);
94 if (unlikely(!bali->lun_alloc_map)) {
95 pr_err("%s: Failed to allocate lun allocation map: "
96 "lun_id=%016llx\n", __func__, ba_lun->lun_id);
97 kfree(bali);
98 return -ENOMEM;
99 }
100
101 /* Initialize the bit map size and set all bits to '1' */
102 bali->free_aun_cnt = lun_size_au;
103
104 for (i = 0; i < bali->lun_bmap_size; i++)
105 bali->lun_alloc_map[i] = 0xFFFFFFFFFFFFFFFFULL;
106
107 /* If the last word not fully utilized, mark extra bits as allocated */
108 last_word_underflow = (bali->lun_bmap_size * BITS_PER_LONG);
109 last_word_underflow -= bali->free_aun_cnt;
110 if (last_word_underflow > 0) {
111 lam = &bali->lun_alloc_map[bali->lun_bmap_size - 1];
112 for (i = (HIBIT - last_word_underflow + 1);
113 i < BITS_PER_LONG;
114 i++)
115 clear_bit(i, (ulong *)lam);
116 }
117
118 /* Initialize high elevator index, low/curr already at 0 from kzalloc */
119 bali->free_high_idx = bali->lun_bmap_size;
120
121 /* Allocate clone map */
122 bali->aun_clone_map = kzalloc((bali->total_aus * sizeof(u8)),
123 GFP_KERNEL);
124 if (unlikely(!bali->aun_clone_map)) {
125 pr_err("%s: Failed to allocate clone map: lun_id=%016llx\n",
126 __func__, ba_lun->lun_id);
127 kfree(bali->lun_alloc_map);
128 kfree(bali);
129 return -ENOMEM;
130 }
131
132 /* Pass the allocated LUN info as a handle to the user */
133 ba_lun->ba_lun_handle = bali;
134
135 pr_debug("%s: Successfully initialized the LUN: "
136 "lun_id=%016llx bitmap size=%x, free_aun_cnt=%llx\n",
137 __func__, ba_lun->lun_id, bali->lun_bmap_size,
138 bali->free_aun_cnt);
139 return 0;
140}
141
142/**
143 * find_free_range() - locates a free bit within the block allocator
144 * @low: First word in block allocator to start search.
145 * @high: Last word in block allocator to search.
146 * @bali: LUN information structure owning the block allocator to search.
147 * @bit_word: Passes back the word in the block allocator owning the free bit.
148 *
149 * Return: The bit position within the passed back word, -1 on failure
150 */
151static int find_free_range(u32 low,
152 u32 high,
153 struct ba_lun_info *bali, int *bit_word)
154{
155 int i;
156 u64 bit_pos = -1;
157 ulong *lam, num_bits;
158
159 for (i = low; i < high; i++)
160 if (bali->lun_alloc_map[i] != 0) {
161 lam = (ulong *)&bali->lun_alloc_map[i];
162 num_bits = (sizeof(*lam) * BITS_PER_BYTE);
163 bit_pos = find_first_bit(lam, num_bits);
164
165 pr_devel("%s: Found free bit %llu in LUN "
166 "map entry %016llx at bitmap index = %d\n",
167 __func__, bit_pos, bali->lun_alloc_map[i], i);
168
169 *bit_word = i;
170 bali->free_aun_cnt--;
171 clear_bit(bit_pos, lam);
172 break;
173 }
174
175 return bit_pos;
176}
177
178/**
179 * ba_alloc() - allocates a block from the block allocator
180 * @ba_lun: Block allocator from which to allocate a block.
181 *
182 * Return: The allocated block, -1 on failure
183 */
184static u64 ba_alloc(struct ba_lun *ba_lun)
185{
186 u64 bit_pos = -1;
187 int bit_word = 0;
188 struct ba_lun_info *bali = NULL;
189
190 bali = ba_lun->ba_lun_handle;
191
192 pr_debug("%s: Received block allocation request: "
193 "lun_id=%016llx free_aun_cnt=%llx\n",
194 __func__, ba_lun->lun_id, bali->free_aun_cnt);
195
196 if (bali->free_aun_cnt == 0) {
197 pr_debug("%s: No space left on LUN: lun_id=%016llx\n",
198 __func__, ba_lun->lun_id);
199 return -1ULL;
200 }
201
202 /* Search to find a free entry, curr->high then low->curr */
203 bit_pos = find_free_range(bali->free_curr_idx,
204 bali->free_high_idx, bali, &bit_word);
205 if (bit_pos == -1) {
206 bit_pos = find_free_range(bali->free_low_idx,
207 bali->free_curr_idx,
208 bali, &bit_word);
209 if (bit_pos == -1) {
210 pr_debug("%s: Could not find an allocation unit on LUN:"
211 " lun_id=%016llx\n", __func__, ba_lun->lun_id);
212 return -1ULL;
213 }
214 }
215
216 /* Update the free_curr_idx */
217 if (bit_pos == HIBIT)
218 bali->free_curr_idx = bit_word + 1;
219 else
220 bali->free_curr_idx = bit_word;
221
222 pr_debug("%s: Allocating AU number=%llx lun_id=%016llx "
223 "free_aun_cnt=%llx\n", __func__,
224 ((bit_word * BITS_PER_LONG) + bit_pos), ba_lun->lun_id,
225 bali->free_aun_cnt);
226
227 return (u64) ((bit_word * BITS_PER_LONG) + bit_pos);
228}
229
230/**
231 * validate_alloc() - validates the specified block has been allocated
232 * @bali: LUN info owning the block allocator.
233 * @aun: Block to validate.
234 *
235 * Return: 0 on success, -1 on failure
236 */
237static int validate_alloc(struct ba_lun_info *bali, u64 aun)
238{
239 int idx = 0, bit_pos = 0;
240
241 idx = aun / BITS_PER_LONG;
242 bit_pos = aun % BITS_PER_LONG;
243
244 if (test_bit(bit_pos, (ulong *)&bali->lun_alloc_map[idx]))
245 return -1;
246
247 return 0;
248}
249
250/**
251 * ba_free() - frees a block from the block allocator
252 * @ba_lun: Block allocator from which to allocate a block.
253 * @to_free: Block to free.
254 *
255 * Return: 0 on success, -1 on failure
256 */
257static int ba_free(struct ba_lun *ba_lun, u64 to_free)
258{
259 int idx = 0, bit_pos = 0;
260 struct ba_lun_info *bali = NULL;
261
262 bali = ba_lun->ba_lun_handle;
263
264 if (validate_alloc(bali, to_free)) {
265 pr_debug("%s: AUN %llx is not allocated on lun_id=%016llx\n",
266 __func__, to_free, ba_lun->lun_id);
267 return -1;
268 }
269
270 pr_debug("%s: Received a request to free AU=%llx lun_id=%016llx "
271 "free_aun_cnt=%llx\n", __func__, to_free, ba_lun->lun_id,
272 bali->free_aun_cnt);
273
274 if (bali->aun_clone_map[to_free] > 0) {
275 pr_debug("%s: AUN %llx lun_id=%016llx cloned. Clone count=%x\n",
276 __func__, to_free, ba_lun->lun_id,
277 bali->aun_clone_map[to_free]);
278 bali->aun_clone_map[to_free]--;
279 return 0;
280 }
281
282 idx = to_free / BITS_PER_LONG;
283 bit_pos = to_free % BITS_PER_LONG;
284
285 set_bit(bit_pos, (ulong *)&bali->lun_alloc_map[idx]);
286 bali->free_aun_cnt++;
287
288 if (idx < bali->free_low_idx)
289 bali->free_low_idx = idx;
290 else if (idx > bali->free_high_idx)
291 bali->free_high_idx = idx;
292
293 pr_debug("%s: Successfully freed AU bit_pos=%x bit map index=%x "
294 "lun_id=%016llx free_aun_cnt=%llx\n", __func__, bit_pos, idx,
295 ba_lun->lun_id, bali->free_aun_cnt);
296
297 return 0;
298}
299
300/**
301 * ba_clone() - Clone a chunk of the block allocation table
302 * @ba_lun: Block allocator from which to allocate a block.
303 * @to_clone: Block to clone.
304 *
305 * Return: 0 on success, -1 on failure
306 */
307static int ba_clone(struct ba_lun *ba_lun, u64 to_clone)
308{
309 struct ba_lun_info *bali = ba_lun->ba_lun_handle;
310
311 if (validate_alloc(bali, to_clone)) {
312 pr_debug("%s: AUN=%llx not allocated on lun_id=%016llx\n",
313 __func__, to_clone, ba_lun->lun_id);
314 return -1;
315 }
316
317 pr_debug("%s: Received a request to clone AUN %llx on lun_id=%016llx\n",
318 __func__, to_clone, ba_lun->lun_id);
319
320 if (bali->aun_clone_map[to_clone] == MAX_AUN_CLONE_CNT) {
321 pr_debug("%s: AUN %llx on lun_id=%016llx hit max clones already\n",
322 __func__, to_clone, ba_lun->lun_id);
323 return -1;
324 }
325
326 bali->aun_clone_map[to_clone]++;
327
328 return 0;
329}
330
331/**
332 * ba_space() - returns the amount of free space left in the block allocator
333 * @ba_lun: Block allocator.
334 *
335 * Return: Amount of free space in block allocator
336 */
337static u64 ba_space(struct ba_lun *ba_lun)
338{
339 struct ba_lun_info *bali = ba_lun->ba_lun_handle;
340
341 return bali->free_aun_cnt;
342}
343
344/**
345 * cxlflash_ba_terminate() - frees resources associated with the block allocator
346 * @ba_lun: Block allocator.
347 *
348 * Safe to call in a partially allocated state.
349 */
350void cxlflash_ba_terminate(struct ba_lun *ba_lun)
351{
352 struct ba_lun_info *bali = ba_lun->ba_lun_handle;
353
354 if (bali) {
355 kfree(bali->aun_clone_map);
356 kfree(bali->lun_alloc_map);
357 kfree(bali);
358 ba_lun->ba_lun_handle = NULL;
359 }
360}
361
362/**
363 * init_vlun() - initializes a LUN for virtual use
364 * @lli: LUN information structure that owns the block allocator.
365 *
366 * Return: 0 on success, -errno on failure
367 */
368static int init_vlun(struct llun_info *lli)
369{
370 int rc = 0;
371 struct glun_info *gli = lli->parent;
372 struct blka *blka = &gli->blka;
373
374 memset(blka, 0, sizeof(*blka));
375 mutex_init(&blka->mutex);
376
377 /* LUN IDs are unique per port, save the index instead */
378 blka->ba_lun.lun_id = lli->lun_index;
379 blka->ba_lun.lsize = gli->max_lba + 1;
380 blka->ba_lun.lba_size = gli->blk_len;
381
382 blka->ba_lun.au_size = MC_CHUNK_SIZE;
383 blka->nchunk = blka->ba_lun.lsize / MC_CHUNK_SIZE;
384
385 rc = ba_init(&blka->ba_lun);
386 if (unlikely(rc))
387 pr_debug("%s: cannot init block_alloc, rc=%d\n", __func__, rc);
388
389 pr_debug("%s: returning rc=%d lli=%p\n", __func__, rc, lli);
390 return rc;
391}
392
393/**
394 * write_same16() - sends a SCSI WRITE_SAME16 (0) command to specified LUN
395 * @sdev: SCSI device associated with LUN.
396 * @lba: Logical block address to start write same.
397 * @nblks: Number of logical blocks to write same.
398 *
399 * The SCSI WRITE_SAME16 can take quite a while to complete. Should an EEH occur
400 * while in scsi_execute_cmd(), the EEH handler will attempt to recover. As
401 * part of the recovery, the handler drains all currently running ioctls,
402 * waiting until they have completed before proceeding with a reset. As this
403 * routine is used on the ioctl path, this can create a condition where the
404 * EEH handler becomes stuck, infinitely waiting for this ioctl thread. To
405 * avoid this behavior, temporarily unmark this thread as an ioctl thread by
406 * releasing the ioctl read semaphore. This will allow the EEH handler to
407 * proceed with a recovery while this thread is still running. Once the
408 * scsi_execute_cmd() returns, reacquire the ioctl read semaphore and check the
409 * adapter state in case it changed while inside of scsi_execute_cmd(). The
410 * state check will wait if the adapter is still being recovered or return a
411 * failure if the recovery failed. In the event that the adapter reset failed,
412 * simply return the failure as the ioctl would be unable to continue.
413 *
414 * Note that the above puts a requirement on this routine to only be called on
415 * an ioctl thread.
416 *
417 * Return: 0 on success, -errno on failure
418 */
419static int write_same16(struct scsi_device *sdev,
420 u64 lba,
421 u32 nblks)
422{
423 u8 *cmd_buf = NULL;
424 u8 *scsi_cmd = NULL;
425 int rc = 0;
426 int result = 0;
427 u64 offset = lba;
428 int left = nblks;
429 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
430 struct device *dev = &cfg->dev->dev;
431 const u32 s = ilog2(sdev->sector_size) - 9;
432 const u32 to = sdev->request_queue->rq_timeout;
433 const u32 ws_limit =
434 sdev->request_queue->limits.max_write_zeroes_sectors >> s;
435
436 cmd_buf = kzalloc(CMD_BUFSIZE, GFP_KERNEL);
437 scsi_cmd = kzalloc(MAX_COMMAND_SIZE, GFP_KERNEL);
438 if (unlikely(!cmd_buf || !scsi_cmd)) {
439 rc = -ENOMEM;
440 goto out;
441 }
442
443 while (left > 0) {
444
445 scsi_cmd[0] = WRITE_SAME_16;
446 scsi_cmd[1] = cfg->ws_unmap ? 0x8 : 0;
447 put_unaligned_be64(offset, &scsi_cmd[2]);
448 put_unaligned_be32(ws_limit < left ? ws_limit : left,
449 &scsi_cmd[10]);
450
451 /* Drop the ioctl read semaphore across lengthy call */
452 up_read(&cfg->ioctl_rwsem);
453 result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_OUT,
454 cmd_buf, CMD_BUFSIZE, to,
455 CMD_RETRIES, NULL);
456 down_read(&cfg->ioctl_rwsem);
457 rc = check_state(cfg);
458 if (rc) {
459 dev_err(dev, "%s: Failed state result=%08x\n",
460 __func__, result);
461 rc = -ENODEV;
462 goto out;
463 }
464
465 if (result) {
466 dev_err_ratelimited(dev, "%s: command failed for "
467 "offset=%lld result=%08x\n",
468 __func__, offset, result);
469 rc = -EIO;
470 goto out;
471 }
472 left -= ws_limit;
473 offset += ws_limit;
474 }
475
476out:
477 kfree(cmd_buf);
478 kfree(scsi_cmd);
479 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
480 return rc;
481}
482
483/**
484 * grow_lxt() - expands the translation table associated with the specified RHTE
485 * @afu: AFU associated with the host.
486 * @sdev: SCSI device associated with LUN.
487 * @ctxid: Context ID of context owning the RHTE.
488 * @rhndl: Resource handle associated with the RHTE.
489 * @rhte: Resource handle entry (RHTE).
490 * @new_size: Number of translation entries associated with RHTE.
491 *
492 * By design, this routine employs a 'best attempt' allocation and will
493 * truncate the requested size down if there is not sufficient space in
494 * the block allocator to satisfy the request but there does exist some
495 * amount of space. The user is made aware of this by returning the size
496 * allocated.
497 *
498 * Return: 0 on success, -errno on failure
499 */
500static int grow_lxt(struct afu *afu,
501 struct scsi_device *sdev,
502 ctx_hndl_t ctxid,
503 res_hndl_t rhndl,
504 struct sisl_rht_entry *rhte,
505 u64 *new_size)
506{
507 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
508 struct device *dev = &cfg->dev->dev;
509 struct sisl_lxt_entry *lxt = NULL, *lxt_old = NULL;
510 struct llun_info *lli = sdev->hostdata;
511 struct glun_info *gli = lli->parent;
512 struct blka *blka = &gli->blka;
513 u32 av_size;
514 u32 ngrps, ngrps_old;
515 u64 aun; /* chunk# allocated by block allocator */
516 u64 delta = *new_size - rhte->lxt_cnt;
517 u64 my_new_size;
518 int i, rc = 0;
519
520 /*
521 * Check what is available in the block allocator before re-allocating
522 * LXT array. This is done up front under the mutex which must not be
523 * released until after allocation is complete.
524 */
525 mutex_lock(&blka->mutex);
526 av_size = ba_space(&blka->ba_lun);
527 if (unlikely(av_size <= 0)) {
528 dev_dbg(dev, "%s: ba_space error av_size=%d\n",
529 __func__, av_size);
530 mutex_unlock(&blka->mutex);
531 rc = -ENOSPC;
532 goto out;
533 }
534
535 if (av_size < delta)
536 delta = av_size;
537
538 lxt_old = rhte->lxt_start;
539 ngrps_old = LXT_NUM_GROUPS(rhte->lxt_cnt);
540 ngrps = LXT_NUM_GROUPS(rhte->lxt_cnt + delta);
541
542 if (ngrps != ngrps_old) {
543 /* reallocate to fit new size */
544 lxt = kzalloc((sizeof(*lxt) * LXT_GROUP_SIZE * ngrps),
545 GFP_KERNEL);
546 if (unlikely(!lxt)) {
547 mutex_unlock(&blka->mutex);
548 rc = -ENOMEM;
549 goto out;
550 }
551
552 /* copy over all old entries */
553 memcpy(lxt, lxt_old, (sizeof(*lxt) * rhte->lxt_cnt));
554 } else
555 lxt = lxt_old;
556
557 /* nothing can fail from now on */
558 my_new_size = rhte->lxt_cnt + delta;
559
560 /* add new entries to the end */
561 for (i = rhte->lxt_cnt; i < my_new_size; i++) {
562 /*
563 * Due to the earlier check of available space, ba_alloc
564 * cannot fail here. If it did due to internal error,
565 * leave a rlba_base of -1u which will likely be a
566 * invalid LUN (too large).
567 */
568 aun = ba_alloc(&blka->ba_lun);
569 if ((aun == -1ULL) || (aun >= blka->nchunk))
570 dev_dbg(dev, "%s: ba_alloc error allocated chunk=%llu "
571 "max=%llu\n", __func__, aun, blka->nchunk - 1);
572
573 /* select both ports, use r/w perms from RHT */
574 lxt[i].rlba_base = ((aun << MC_CHUNK_SHIFT) |
575 (lli->lun_index << LXT_LUNIDX_SHIFT) |
576 (RHT_PERM_RW << LXT_PERM_SHIFT |
577 lli->port_sel));
578 }
579
580 mutex_unlock(&blka->mutex);
581
582 /*
583 * The following sequence is prescribed in the SISlite spec
584 * for syncing up with the AFU when adding LXT entries.
585 */
586 dma_wmb(); /* Make LXT updates are visible */
587
588 rhte->lxt_start = lxt;
589 dma_wmb(); /* Make RHT entry's LXT table update visible */
590
591 rhte->lxt_cnt = my_new_size;
592 dma_wmb(); /* Make RHT entry's LXT table size update visible */
593
594 rc = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_LW_SYNC);
595 if (unlikely(rc))
596 rc = -EAGAIN;
597
598 /* free old lxt if reallocated */
599 if (lxt != lxt_old)
600 kfree(lxt_old);
601 *new_size = my_new_size;
602out:
603 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
604 return rc;
605}
606
607/**
608 * shrink_lxt() - reduces translation table associated with the specified RHTE
609 * @afu: AFU associated with the host.
610 * @sdev: SCSI device associated with LUN.
611 * @rhndl: Resource handle associated with the RHTE.
612 * @rhte: Resource handle entry (RHTE).
613 * @ctxi: Context owning resources.
614 * @new_size: Number of translation entries associated with RHTE.
615 *
616 * Return: 0 on success, -errno on failure
617 */
618static int shrink_lxt(struct afu *afu,
619 struct scsi_device *sdev,
620 res_hndl_t rhndl,
621 struct sisl_rht_entry *rhte,
622 struct ctx_info *ctxi,
623 u64 *new_size)
624{
625 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
626 struct device *dev = &cfg->dev->dev;
627 struct sisl_lxt_entry *lxt, *lxt_old;
628 struct llun_info *lli = sdev->hostdata;
629 struct glun_info *gli = lli->parent;
630 struct blka *blka = &gli->blka;
631 ctx_hndl_t ctxid = DECODE_CTXID(ctxi->ctxid);
632 bool needs_ws = ctxi->rht_needs_ws[rhndl];
633 bool needs_sync = !ctxi->err_recovery_active;
634 u32 ngrps, ngrps_old;
635 u64 aun; /* chunk# allocated by block allocator */
636 u64 delta = rhte->lxt_cnt - *new_size;
637 u64 my_new_size;
638 int i, rc = 0;
639
640 lxt_old = rhte->lxt_start;
641 ngrps_old = LXT_NUM_GROUPS(rhte->lxt_cnt);
642 ngrps = LXT_NUM_GROUPS(rhte->lxt_cnt - delta);
643
644 if (ngrps != ngrps_old) {
645 /* Reallocate to fit new size unless new size is 0 */
646 if (ngrps) {
647 lxt = kzalloc((sizeof(*lxt) * LXT_GROUP_SIZE * ngrps),
648 GFP_KERNEL);
649 if (unlikely(!lxt)) {
650 rc = -ENOMEM;
651 goto out;
652 }
653
654 /* Copy over old entries that will remain */
655 memcpy(lxt, lxt_old,
656 (sizeof(*lxt) * (rhte->lxt_cnt - delta)));
657 } else
658 lxt = NULL;
659 } else
660 lxt = lxt_old;
661
662 /* Nothing can fail from now on */
663 my_new_size = rhte->lxt_cnt - delta;
664
665 /*
666 * The following sequence is prescribed in the SISlite spec
667 * for syncing up with the AFU when removing LXT entries.
668 */
669 rhte->lxt_cnt = my_new_size;
670 dma_wmb(); /* Make RHT entry's LXT table size update visible */
671
672 rhte->lxt_start = lxt;
673 dma_wmb(); /* Make RHT entry's LXT table update visible */
674
675 if (needs_sync) {
676 rc = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);
677 if (unlikely(rc))
678 rc = -EAGAIN;
679 }
680
681 if (needs_ws) {
682 /*
683 * Mark the context as unavailable, so that we can release
684 * the mutex safely.
685 */
686 ctxi->unavail = true;
687 mutex_unlock(&ctxi->mutex);
688 }
689
690 /* Free LBAs allocated to freed chunks */
691 mutex_lock(&blka->mutex);
692 for (i = delta - 1; i >= 0; i--) {
693 aun = lxt_old[my_new_size + i].rlba_base >> MC_CHUNK_SHIFT;
694 if (needs_ws)
695 write_same16(sdev, aun, MC_CHUNK_SIZE);
696 ba_free(&blka->ba_lun, aun);
697 }
698 mutex_unlock(&blka->mutex);
699
700 if (needs_ws) {
701 /* Make the context visible again */
702 mutex_lock(&ctxi->mutex);
703 ctxi->unavail = false;
704 }
705
706 /* Free old lxt if reallocated */
707 if (lxt != lxt_old)
708 kfree(lxt_old);
709 *new_size = my_new_size;
710out:
711 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
712 return rc;
713}
714
715/**
716 * _cxlflash_vlun_resize() - changes the size of a virtual LUN
717 * @sdev: SCSI device associated with LUN owning virtual LUN.
718 * @ctxi: Context owning resources.
719 * @resize: Resize ioctl data structure.
720 *
721 * On successful return, the user is informed of the new size (in blocks)
722 * of the virtual LUN in last LBA format. When the size of the virtual
723 * LUN is zero, the last LBA is reflected as -1. See comment in the
724 * prologue for _cxlflash_disk_release() regarding AFU syncs and contexts
725 * on the error recovery list.
726 *
727 * Return: 0 on success, -errno on failure
728 */
729int _cxlflash_vlun_resize(struct scsi_device *sdev,
730 struct ctx_info *ctxi,
731 struct dk_cxlflash_resize *resize)
732{
733 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
734 struct device *dev = &cfg->dev->dev;
735 struct llun_info *lli = sdev->hostdata;
736 struct glun_info *gli = lli->parent;
737 struct afu *afu = cfg->afu;
738 bool put_ctx = false;
739
740 res_hndl_t rhndl = resize->rsrc_handle;
741 u64 new_size;
742 u64 nsectors;
743 u64 ctxid = DECODE_CTXID(resize->context_id),
744 rctxid = resize->context_id;
745
746 struct sisl_rht_entry *rhte;
747
748 int rc = 0;
749
750 /*
751 * The requested size (req_size) is always assumed to be in 4k blocks,
752 * so we have to convert it here from 4k to chunk size.
753 */
754 nsectors = (resize->req_size * CXLFLASH_BLOCK_SIZE) / gli->blk_len;
755 new_size = DIV_ROUND_UP(nsectors, MC_CHUNK_SIZE);
756
757 dev_dbg(dev, "%s: ctxid=%llu rhndl=%llu req_size=%llu new_size=%llu\n",
758 __func__, ctxid, resize->rsrc_handle, resize->req_size,
759 new_size);
760
761 if (unlikely(gli->mode != MODE_VIRTUAL)) {
762 dev_dbg(dev, "%s: LUN mode does not support resize mode=%d\n",
763 __func__, gli->mode);
764 rc = -EINVAL;
765 goto out;
766
767 }
768
769 if (!ctxi) {
770 ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
771 if (unlikely(!ctxi)) {
772 dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
773 __func__, ctxid);
774 rc = -EINVAL;
775 goto out;
776 }
777
778 put_ctx = true;
779 }
780
781 rhte = get_rhte(ctxi, rhndl, lli);
782 if (unlikely(!rhte)) {
783 dev_dbg(dev, "%s: Bad resource handle rhndl=%u\n",
784 __func__, rhndl);
785 rc = -EINVAL;
786 goto out;
787 }
788
789 if (new_size > rhte->lxt_cnt)
790 rc = grow_lxt(afu, sdev, ctxid, rhndl, rhte, &new_size);
791 else if (new_size < rhte->lxt_cnt)
792 rc = shrink_lxt(afu, sdev, rhndl, rhte, ctxi, &new_size);
793 else {
794 /*
795 * Rare case where there is already sufficient space, just
796 * need to perform a translation sync with the AFU. This
797 * scenario likely follows a previous sync failure during
798 * a resize operation. Accordingly, perform the heavyweight
799 * form of translation sync as it is unknown which type of
800 * resize failed previously.
801 */
802 rc = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);
803 if (unlikely(rc)) {
804 rc = -EAGAIN;
805 goto out;
806 }
807 }
808
809 resize->hdr.return_flags = 0;
810 resize->last_lba = (new_size * MC_CHUNK_SIZE * gli->blk_len);
811 resize->last_lba /= CXLFLASH_BLOCK_SIZE;
812 resize->last_lba--;
813
814out:
815 if (put_ctx)
816 put_context(ctxi);
817 dev_dbg(dev, "%s: resized to %llu returning rc=%d\n",
818 __func__, resize->last_lba, rc);
819 return rc;
820}
821
822int cxlflash_vlun_resize(struct scsi_device *sdev,
823 struct dk_cxlflash_resize *resize)
824{
825 return _cxlflash_vlun_resize(sdev, NULL, resize);
826}
827
828/**
829 * cxlflash_restore_luntable() - Restore LUN table to prior state
830 * @cfg: Internal structure associated with the host.
831 */
832void cxlflash_restore_luntable(struct cxlflash_cfg *cfg)
833{
834 struct llun_info *lli, *temp;
835 u32 lind;
836 int k;
837 struct device *dev = &cfg->dev->dev;
838 __be64 __iomem *fc_port_luns;
839
840 mutex_lock(&global.mutex);
841
842 list_for_each_entry_safe(lli, temp, &cfg->lluns, list) {
843 if (!lli->in_table)
844 continue;
845
846 lind = lli->lun_index;
847 dev_dbg(dev, "%s: Virtual LUNs on slot %d:\n", __func__, lind);
848
849 for (k = 0; k < cfg->num_fc_ports; k++)
850 if (lli->port_sel & (1 << k)) {
851 fc_port_luns = get_fc_port_luns(cfg, k);
852 writeq_be(lli->lun_id[k], &fc_port_luns[lind]);
853 dev_dbg(dev, "\t%d=%llx\n", k, lli->lun_id[k]);
854 }
855 }
856
857 mutex_unlock(&global.mutex);
858}
859
860/**
861 * get_num_ports() - compute number of ports from port selection mask
862 * @psm: Port selection mask.
863 *
864 * Return: Population count of port selection mask
865 */
866static inline u8 get_num_ports(u32 psm)
867{
868 static const u8 bits[16] = { 0, 1, 1, 2, 1, 2, 2, 3,
869 1, 2, 2, 3, 2, 3, 3, 4 };
870
871 return bits[psm & 0xf];
872}
873
874/**
875 * init_luntable() - write an entry in the LUN table
876 * @cfg: Internal structure associated with the host.
877 * @lli: Per adapter LUN information structure.
878 *
879 * On successful return, a LUN table entry is created:
880 * - at the top for LUNs visible on multiple ports.
881 * - at the bottom for LUNs visible only on one port.
882 *
883 * Return: 0 on success, -errno on failure
884 */
885static int init_luntable(struct cxlflash_cfg *cfg, struct llun_info *lli)
886{
887 u32 chan;
888 u32 lind;
889 u32 nports;
890 int rc = 0;
891 int k;
892 struct device *dev = &cfg->dev->dev;
893 __be64 __iomem *fc_port_luns;
894
895 mutex_lock(&global.mutex);
896
897 if (lli->in_table)
898 goto out;
899
900 nports = get_num_ports(lli->port_sel);
901 if (nports == 0 || nports > cfg->num_fc_ports) {
902 WARN(1, "Unsupported port configuration nports=%u", nports);
903 rc = -EIO;
904 goto out;
905 }
906
907 if (nports > 1) {
908 /*
909 * When LUN is visible from multiple ports, we will put
910 * it in the top half of the LUN table.
911 */
912 for (k = 0; k < cfg->num_fc_ports; k++) {
913 if (!(lli->port_sel & (1 << k)))
914 continue;
915
916 if (cfg->promote_lun_index == cfg->last_lun_index[k]) {
917 rc = -ENOSPC;
918 goto out;
919 }
920 }
921
922 lind = lli->lun_index = cfg->promote_lun_index;
923 dev_dbg(dev, "%s: Virtual LUNs on slot %d:\n", __func__, lind);
924
925 for (k = 0; k < cfg->num_fc_ports; k++) {
926 if (!(lli->port_sel & (1 << k)))
927 continue;
928
929 fc_port_luns = get_fc_port_luns(cfg, k);
930 writeq_be(lli->lun_id[k], &fc_port_luns[lind]);
931 dev_dbg(dev, "\t%d=%llx\n", k, lli->lun_id[k]);
932 }
933
934 cfg->promote_lun_index++;
935 } else {
936 /*
937 * When LUN is visible only from one port, we will put
938 * it in the bottom half of the LUN table.
939 */
940 chan = PORTMASK2CHAN(lli->port_sel);
941 if (cfg->promote_lun_index == cfg->last_lun_index[chan]) {
942 rc = -ENOSPC;
943 goto out;
944 }
945
946 lind = lli->lun_index = cfg->last_lun_index[chan];
947 fc_port_luns = get_fc_port_luns(cfg, chan);
948 writeq_be(lli->lun_id[chan], &fc_port_luns[lind]);
949 cfg->last_lun_index[chan]--;
950 dev_dbg(dev, "%s: Virtual LUNs on slot %d:\n\t%d=%llx\n",
951 __func__, lind, chan, lli->lun_id[chan]);
952 }
953
954 lli->in_table = true;
955out:
956 mutex_unlock(&global.mutex);
957 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
958 return rc;
959}
960
961/**
962 * cxlflash_disk_virtual_open() - open a virtual disk of specified size
963 * @sdev: SCSI device associated with LUN owning virtual LUN.
964 * @arg: UVirtual ioctl data structure.
965 *
966 * On successful return, the user is informed of the resource handle
967 * to be used to identify the virtual LUN and the size (in blocks) of
968 * the virtual LUN in last LBA format. When the size of the virtual LUN
969 * is zero, the last LBA is reflected as -1.
970 *
971 * Return: 0 on success, -errno on failure
972 */
973int cxlflash_disk_virtual_open(struct scsi_device *sdev, void *arg)
974{
975 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
976 struct device *dev = &cfg->dev->dev;
977 struct llun_info *lli = sdev->hostdata;
978 struct glun_info *gli = lli->parent;
979
980 struct dk_cxlflash_uvirtual *virt = (struct dk_cxlflash_uvirtual *)arg;
981 struct dk_cxlflash_resize resize;
982
983 u64 ctxid = DECODE_CTXID(virt->context_id),
984 rctxid = virt->context_id;
985 u64 lun_size = virt->lun_size;
986 u64 last_lba = 0;
987 u64 rsrc_handle = -1;
988
989 int rc = 0;
990
991 struct ctx_info *ctxi = NULL;
992 struct sisl_rht_entry *rhte = NULL;
993
994 dev_dbg(dev, "%s: ctxid=%llu ls=%llu\n", __func__, ctxid, lun_size);
995
996 /* Setup the LUNs block allocator on first call */
997 mutex_lock(&gli->mutex);
998 if (gli->mode == MODE_NONE) {
999 rc = init_vlun(lli);
1000 if (rc) {
1001 dev_err(dev, "%s: init_vlun failed rc=%d\n",
1002 __func__, rc);
1003 rc = -ENOMEM;
1004 goto err0;
1005 }
1006 }
1007
1008 rc = cxlflash_lun_attach(gli, MODE_VIRTUAL, true);
1009 if (unlikely(rc)) {
1010 dev_err(dev, "%s: Failed attach to LUN (VIRTUAL)\n", __func__);
1011 goto err0;
1012 }
1013 mutex_unlock(&gli->mutex);
1014
1015 rc = init_luntable(cfg, lli);
1016 if (rc) {
1017 dev_err(dev, "%s: init_luntable failed rc=%d\n", __func__, rc);
1018 goto err1;
1019 }
1020
1021 ctxi = get_context(cfg, rctxid, lli, 0);
1022 if (unlikely(!ctxi)) {
1023 dev_err(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1024 rc = -EINVAL;
1025 goto err1;
1026 }
1027
1028 rhte = rhte_checkout(ctxi, lli);
1029 if (unlikely(!rhte)) {
1030 dev_err(dev, "%s: too many opens ctxid=%llu\n",
1031 __func__, ctxid);
1032 rc = -EMFILE; /* too many opens */
1033 goto err1;
1034 }
1035
1036 rsrc_handle = (rhte - ctxi->rht_start);
1037
1038 /* Populate RHT format 0 */
1039 rhte->nmask = MC_RHT_NMASK;
1040 rhte->fp = SISL_RHT_FP(0U, ctxi->rht_perms);
1041
1042 /* Resize even if requested size is 0 */
1043 marshal_virt_to_resize(virt, &resize);
1044 resize.rsrc_handle = rsrc_handle;
1045 rc = _cxlflash_vlun_resize(sdev, ctxi, &resize);
1046 if (rc) {
1047 dev_err(dev, "%s: resize failed rc=%d\n", __func__, rc);
1048 goto err2;
1049 }
1050 last_lba = resize.last_lba;
1051
1052 if (virt->hdr.flags & DK_CXLFLASH_UVIRTUAL_NEED_WRITE_SAME)
1053 ctxi->rht_needs_ws[rsrc_handle] = true;
1054
1055 virt->hdr.return_flags = 0;
1056 virt->last_lba = last_lba;
1057 virt->rsrc_handle = rsrc_handle;
1058
1059 if (get_num_ports(lli->port_sel) > 1)
1060 virt->hdr.return_flags |= DK_CXLFLASH_ALL_PORTS_ACTIVE;
1061out:
1062 if (likely(ctxi))
1063 put_context(ctxi);
1064 dev_dbg(dev, "%s: returning handle=%llu rc=%d llba=%llu\n",
1065 __func__, rsrc_handle, rc, last_lba);
1066 return rc;
1067
1068err2:
1069 rhte_checkin(ctxi, rhte);
1070err1:
1071 cxlflash_lun_detach(gli);
1072 goto out;
1073err0:
1074 /* Special common cleanup prior to successful LUN attach */
1075 cxlflash_ba_terminate(&gli->blka.ba_lun);
1076 mutex_unlock(&gli->mutex);
1077 goto out;
1078}
1079
1080/**
1081 * clone_lxt() - copies translation tables from source to destination RHTE
1082 * @afu: AFU associated with the host.
1083 * @blka: Block allocator associated with LUN.
1084 * @ctxid: Context ID of context owning the RHTE.
1085 * @rhndl: Resource handle associated with the RHTE.
1086 * @rhte: Destination resource handle entry (RHTE).
1087 * @rhte_src: Source resource handle entry (RHTE).
1088 *
1089 * Return: 0 on success, -errno on failure
1090 */
1091static int clone_lxt(struct afu *afu,
1092 struct blka *blka,
1093 ctx_hndl_t ctxid,
1094 res_hndl_t rhndl,
1095 struct sisl_rht_entry *rhte,
1096 struct sisl_rht_entry *rhte_src)
1097{
1098 struct cxlflash_cfg *cfg = afu->parent;
1099 struct device *dev = &cfg->dev->dev;
1100 struct sisl_lxt_entry *lxt = NULL;
1101 bool locked = false;
1102 u32 ngrps;
1103 u64 aun; /* chunk# allocated by block allocator */
1104 int j;
1105 int i = 0;
1106 int rc = 0;
1107
1108 ngrps = LXT_NUM_GROUPS(rhte_src->lxt_cnt);
1109
1110 if (ngrps) {
1111 /* allocate new LXTs for clone */
1112 lxt = kzalloc((sizeof(*lxt) * LXT_GROUP_SIZE * ngrps),
1113 GFP_KERNEL);
1114 if (unlikely(!lxt)) {
1115 rc = -ENOMEM;
1116 goto out;
1117 }
1118
1119 /* copy over */
1120 memcpy(lxt, rhte_src->lxt_start,
1121 (sizeof(*lxt) * rhte_src->lxt_cnt));
1122
1123 /* clone the LBAs in block allocator via ref_cnt, note that the
1124 * block allocator mutex must be held until it is established
1125 * that this routine will complete without the need for a
1126 * cleanup.
1127 */
1128 mutex_lock(&blka->mutex);
1129 locked = true;
1130 for (i = 0; i < rhte_src->lxt_cnt; i++) {
1131 aun = (lxt[i].rlba_base >> MC_CHUNK_SHIFT);
1132 if (ba_clone(&blka->ba_lun, aun) == -1ULL) {
1133 rc = -EIO;
1134 goto err;
1135 }
1136 }
1137 }
1138
1139 /*
1140 * The following sequence is prescribed in the SISlite spec
1141 * for syncing up with the AFU when adding LXT entries.
1142 */
1143 dma_wmb(); /* Make LXT updates are visible */
1144
1145 rhte->lxt_start = lxt;
1146 dma_wmb(); /* Make RHT entry's LXT table update visible */
1147
1148 rhte->lxt_cnt = rhte_src->lxt_cnt;
1149 dma_wmb(); /* Make RHT entry's LXT table size update visible */
1150
1151 rc = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_LW_SYNC);
1152 if (unlikely(rc)) {
1153 rc = -EAGAIN;
1154 goto err2;
1155 }
1156
1157out:
1158 if (locked)
1159 mutex_unlock(&blka->mutex);
1160 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
1161 return rc;
1162err2:
1163 /* Reset the RHTE */
1164 rhte->lxt_cnt = 0;
1165 dma_wmb();
1166 rhte->lxt_start = NULL;
1167 dma_wmb();
1168err:
1169 /* free the clones already made */
1170 for (j = 0; j < i; j++) {
1171 aun = (lxt[j].rlba_base >> MC_CHUNK_SHIFT);
1172 ba_free(&blka->ba_lun, aun);
1173 }
1174 kfree(lxt);
1175 goto out;
1176}
1177
1178/**
1179 * cxlflash_disk_clone() - clone a context by making snapshot of another
1180 * @sdev: SCSI device associated with LUN owning virtual LUN.
1181 * @clone: Clone ioctl data structure.
1182 *
1183 * This routine effectively performs cxlflash_disk_open operation for each
1184 * in-use virtual resource in the source context. Note that the destination
1185 * context must be in pristine state and cannot have any resource handles
1186 * open at the time of the clone.
1187 *
1188 * Return: 0 on success, -errno on failure
1189 */
1190int cxlflash_disk_clone(struct scsi_device *sdev,
1191 struct dk_cxlflash_clone *clone)
1192{
1193 struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1194 struct device *dev = &cfg->dev->dev;
1195 struct llun_info *lli = sdev->hostdata;
1196 struct glun_info *gli = lli->parent;
1197 struct blka *blka = &gli->blka;
1198 struct afu *afu = cfg->afu;
1199 struct dk_cxlflash_release release = { { 0 }, 0 };
1200
1201 struct ctx_info *ctxi_src = NULL,
1202 *ctxi_dst = NULL;
1203 struct lun_access *lun_access_src, *lun_access_dst;
1204 u32 perms;
1205 u64 ctxid_src = DECODE_CTXID(clone->context_id_src),
1206 ctxid_dst = DECODE_CTXID(clone->context_id_dst),
1207 rctxid_src = clone->context_id_src,
1208 rctxid_dst = clone->context_id_dst;
1209 int i, j;
1210 int rc = 0;
1211 bool found;
1212 LIST_HEAD(sidecar);
1213
1214 dev_dbg(dev, "%s: ctxid_src=%llu ctxid_dst=%llu\n",
1215 __func__, ctxid_src, ctxid_dst);
1216
1217 /* Do not clone yourself */
1218 if (unlikely(rctxid_src == rctxid_dst)) {
1219 rc = -EINVAL;
1220 goto out;
1221 }
1222
1223 if (unlikely(gli->mode != MODE_VIRTUAL)) {
1224 rc = -EINVAL;
1225 dev_dbg(dev, "%s: Only supported on virtual LUNs mode=%u\n",
1226 __func__, gli->mode);
1227 goto out;
1228 }
1229
1230 ctxi_src = get_context(cfg, rctxid_src, lli, CTX_CTRL_CLONE);
1231 ctxi_dst = get_context(cfg, rctxid_dst, lli, 0);
1232 if (unlikely(!ctxi_src || !ctxi_dst)) {
1233 dev_dbg(dev, "%s: Bad context ctxid_src=%llu ctxid_dst=%llu\n",
1234 __func__, ctxid_src, ctxid_dst);
1235 rc = -EINVAL;
1236 goto out;
1237 }
1238
1239 /* Verify there is no open resource handle in the destination context */
1240 for (i = 0; i < MAX_RHT_PER_CONTEXT; i++)
1241 if (ctxi_dst->rht_start[i].nmask != 0) {
1242 rc = -EINVAL;
1243 goto out;
1244 }
1245
1246 /* Clone LUN access list */
1247 list_for_each_entry(lun_access_src, &ctxi_src->luns, list) {
1248 found = false;
1249 list_for_each_entry(lun_access_dst, &ctxi_dst->luns, list)
1250 if (lun_access_dst->sdev == lun_access_src->sdev) {
1251 found = true;
1252 break;
1253 }
1254
1255 if (!found) {
1256 lun_access_dst = kzalloc(sizeof(*lun_access_dst),
1257 GFP_KERNEL);
1258 if (unlikely(!lun_access_dst)) {
1259 dev_err(dev, "%s: lun_access allocation fail\n",
1260 __func__);
1261 rc = -ENOMEM;
1262 goto out;
1263 }
1264
1265 *lun_access_dst = *lun_access_src;
1266 list_add(&lun_access_dst->list, &sidecar);
1267 }
1268 }
1269
1270 if (unlikely(!ctxi_src->rht_out)) {
1271 dev_dbg(dev, "%s: Nothing to clone\n", __func__);
1272 goto out_success;
1273 }
1274
1275 /* User specified permission on attach */
1276 perms = ctxi_dst->rht_perms;
1277
1278 /*
1279 * Copy over checked-out RHT (and their associated LXT) entries by
1280 * hand, stopping after we've copied all outstanding entries and
1281 * cleaning up if the clone fails.
1282 *
1283 * Note: This loop is equivalent to performing cxlflash_disk_open and
1284 * cxlflash_vlun_resize. As such, LUN accounting needs to be taken into
1285 * account by attaching after each successful RHT entry clone. In the
1286 * event that a clone failure is experienced, the LUN detach is handled
1287 * via the cleanup performed by _cxlflash_disk_release.
1288 */
1289 for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) {
1290 if (ctxi_src->rht_out == ctxi_dst->rht_out)
1291 break;
1292 if (ctxi_src->rht_start[i].nmask == 0)
1293 continue;
1294
1295 /* Consume a destination RHT entry */
1296 ctxi_dst->rht_out++;
1297 ctxi_dst->rht_start[i].nmask = ctxi_src->rht_start[i].nmask;
1298 ctxi_dst->rht_start[i].fp =
1299 SISL_RHT_FP_CLONE(ctxi_src->rht_start[i].fp, perms);
1300 ctxi_dst->rht_lun[i] = ctxi_src->rht_lun[i];
1301
1302 rc = clone_lxt(afu, blka, ctxid_dst, i,
1303 &ctxi_dst->rht_start[i],
1304 &ctxi_src->rht_start[i]);
1305 if (rc) {
1306 marshal_clone_to_rele(clone, &release);
1307 for (j = 0; j < i; j++) {
1308 release.rsrc_handle = j;
1309 _cxlflash_disk_release(sdev, ctxi_dst,
1310 &release);
1311 }
1312
1313 /* Put back the one we failed on */
1314 rhte_checkin(ctxi_dst, &ctxi_dst->rht_start[i]);
1315 goto err;
1316 }
1317
1318 cxlflash_lun_attach(gli, gli->mode, false);
1319 }
1320
1321out_success:
1322 list_splice(&sidecar, &ctxi_dst->luns);
1323
1324 /* fall through */
1325out:
1326 if (ctxi_src)
1327 put_context(ctxi_src);
1328 if (ctxi_dst)
1329 put_context(ctxi_dst);
1330 dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
1331 return rc;
1332
1333err:
1334 list_for_each_entry_safe(lun_access_src, lun_access_dst, &sidecar, list)
1335 kfree(lun_access_src);
1336 goto out;
1337}