Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2015 IBM Corp.
4 */
5
6#include <linux/spinlock.h>
7#include <linux/uaccess.h>
8#include <linux/delay.h>
9
10#include "cxl.h"
11#include "hcalls.h"
12#include "trace.h"
13
14#define CXL_ERROR_DETECTED_EVENT 1
15#define CXL_SLOT_RESET_EVENT 2
16#define CXL_RESUME_EVENT 3
17
18static void pci_error_handlers(struct cxl_afu *afu,
19 int bus_error_event,
20 pci_channel_state_t state)
21{
22 struct pci_dev *afu_dev;
23
24 if (afu->phb == NULL)
25 return;
26
27 list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {
28 if (!afu_dev->driver)
29 continue;
30
31 switch (bus_error_event) {
32 case CXL_ERROR_DETECTED_EVENT:
33 afu_dev->error_state = state;
34
35 if (afu_dev->driver->err_handler &&
36 afu_dev->driver->err_handler->error_detected)
37 afu_dev->driver->err_handler->error_detected(afu_dev, state);
38 break;
39 case CXL_SLOT_RESET_EVENT:
40 afu_dev->error_state = state;
41
42 if (afu_dev->driver->err_handler &&
43 afu_dev->driver->err_handler->slot_reset)
44 afu_dev->driver->err_handler->slot_reset(afu_dev);
45 break;
46 case CXL_RESUME_EVENT:
47 if (afu_dev->driver->err_handler &&
48 afu_dev->driver->err_handler->resume)
49 afu_dev->driver->err_handler->resume(afu_dev);
50 break;
51 }
52 }
53}
54
55static irqreturn_t guest_handle_psl_slice_error(struct cxl_context *ctx, u64 dsisr,
56 u64 errstat)
57{
58 pr_devel("in %s\n", __func__);
59 dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%.16llx\n", errstat);
60
61 return cxl_ops->ack_irq(ctx, 0, errstat);
62}
63
64static ssize_t guest_collect_vpd(struct cxl *adapter, struct cxl_afu *afu,
65 void *buf, size_t len)
66{
67 unsigned int entries, mod;
68 unsigned long **vpd_buf = NULL;
69 struct sg_list *le;
70 int rc = 0, i, tocopy;
71 u64 out = 0;
72
73 if (buf == NULL)
74 return -EINVAL;
75
76 /* number of entries in the list */
77 entries = len / SG_BUFFER_SIZE;
78 mod = len % SG_BUFFER_SIZE;
79 if (mod)
80 entries++;
81
82 if (entries > SG_MAX_ENTRIES) {
83 entries = SG_MAX_ENTRIES;
84 len = SG_MAX_ENTRIES * SG_BUFFER_SIZE;
85 mod = 0;
86 }
87
88 vpd_buf = kcalloc(entries, sizeof(unsigned long *), GFP_KERNEL);
89 if (!vpd_buf)
90 return -ENOMEM;
91
92 le = (struct sg_list *)get_zeroed_page(GFP_KERNEL);
93 if (!le) {
94 rc = -ENOMEM;
95 goto err1;
96 }
97
98 for (i = 0; i < entries; i++) {
99 vpd_buf[i] = (unsigned long *)get_zeroed_page(GFP_KERNEL);
100 if (!vpd_buf[i]) {
101 rc = -ENOMEM;
102 goto err2;
103 }
104 le[i].phys_addr = cpu_to_be64(virt_to_phys(vpd_buf[i]));
105 le[i].len = cpu_to_be64(SG_BUFFER_SIZE);
106 if ((i == (entries - 1)) && mod)
107 le[i].len = cpu_to_be64(mod);
108 }
109
110 if (adapter)
111 rc = cxl_h_collect_vpd_adapter(adapter->guest->handle,
112 virt_to_phys(le), entries, &out);
113 else
114 rc = cxl_h_collect_vpd(afu->guest->handle, 0,
115 virt_to_phys(le), entries, &out);
116 pr_devel("length of available (entries: %i), vpd: %#llx\n",
117 entries, out);
118
119 if (!rc) {
120 /*
121 * hcall returns in 'out' the size of available VPDs.
122 * It fills the buffer with as much data as possible.
123 */
124 if (out < len)
125 len = out;
126 rc = len;
127 if (out) {
128 for (i = 0; i < entries; i++) {
129 if (len < SG_BUFFER_SIZE)
130 tocopy = len;
131 else
132 tocopy = SG_BUFFER_SIZE;
133 memcpy(buf, vpd_buf[i], tocopy);
134 buf += tocopy;
135 len -= tocopy;
136 }
137 }
138 }
139err2:
140 for (i = 0; i < entries; i++) {
141 if (vpd_buf[i])
142 free_page((unsigned long) vpd_buf[i]);
143 }
144 free_page((unsigned long) le);
145err1:
146 kfree(vpd_buf);
147 return rc;
148}
149
150static int guest_get_irq_info(struct cxl_context *ctx, struct cxl_irq_info *info)
151{
152 return cxl_h_collect_int_info(ctx->afu->guest->handle, ctx->process_token, info);
153}
154
155static irqreturn_t guest_psl_irq(int irq, void *data)
156{
157 struct cxl_context *ctx = data;
158 struct cxl_irq_info irq_info;
159 int rc;
160
161 pr_devel("%d: received PSL interrupt %i\n", ctx->pe, irq);
162 rc = guest_get_irq_info(ctx, &irq_info);
163 if (rc) {
164 WARN(1, "Unable to get IRQ info: %i\n", rc);
165 return IRQ_HANDLED;
166 }
167
168 rc = cxl_irq_psl8(irq, ctx, &irq_info);
169 return rc;
170}
171
172static int afu_read_error_state(struct cxl_afu *afu, int *state_out)
173{
174 u64 state;
175 int rc = 0;
176
177 if (!afu)
178 return -EIO;
179
180 rc = cxl_h_read_error_state(afu->guest->handle, &state);
181 if (!rc) {
182 WARN_ON(state != H_STATE_NORMAL &&
183 state != H_STATE_DISABLE &&
184 state != H_STATE_TEMP_UNAVAILABLE &&
185 state != H_STATE_PERM_UNAVAILABLE);
186 *state_out = state & 0xffffffff;
187 }
188 return rc;
189}
190
191static irqreturn_t guest_slice_irq_err(int irq, void *data)
192{
193 struct cxl_afu *afu = data;
194 int rc;
195 u64 serr, afu_error, dsisr;
196
197 rc = cxl_h_get_fn_error_interrupt(afu->guest->handle, &serr);
198 if (rc) {
199 dev_crit(&afu->dev, "Couldn't read PSL_SERR_An: %d\n", rc);
200 return IRQ_HANDLED;
201 }
202 afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An);
203 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
204 cxl_afu_decode_psl_serr(afu, serr);
205 dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error);
206 dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr);
207
208 rc = cxl_h_ack_fn_error_interrupt(afu->guest->handle, serr);
209 if (rc)
210 dev_crit(&afu->dev, "Couldn't ack slice error interrupt: %d\n",
211 rc);
212
213 return IRQ_HANDLED;
214}
215
216
217static int irq_alloc_range(struct cxl *adapter, int len, int *irq)
218{
219 int i, n;
220 struct irq_avail *cur;
221
222 for (i = 0; i < adapter->guest->irq_nranges; i++) {
223 cur = &adapter->guest->irq_avail[i];
224 n = bitmap_find_next_zero_area(cur->bitmap, cur->range,
225 0, len, 0);
226 if (n < cur->range) {
227 bitmap_set(cur->bitmap, n, len);
228 *irq = cur->offset + n;
229 pr_devel("guest: allocate IRQs %#x->%#x\n",
230 *irq, *irq + len - 1);
231
232 return 0;
233 }
234 }
235 return -ENOSPC;
236}
237
238static int irq_free_range(struct cxl *adapter, int irq, int len)
239{
240 int i, n;
241 struct irq_avail *cur;
242
243 if (len == 0)
244 return -ENOENT;
245
246 for (i = 0; i < adapter->guest->irq_nranges; i++) {
247 cur = &adapter->guest->irq_avail[i];
248 if (irq >= cur->offset &&
249 (irq + len) <= (cur->offset + cur->range)) {
250 n = irq - cur->offset;
251 bitmap_clear(cur->bitmap, n, len);
252 pr_devel("guest: release IRQs %#x->%#x\n",
253 irq, irq + len - 1);
254 return 0;
255 }
256 }
257 return -ENOENT;
258}
259
260static int guest_reset(struct cxl *adapter)
261{
262 struct cxl_afu *afu = NULL;
263 int i, rc;
264
265 pr_devel("Adapter reset request\n");
266 spin_lock(&adapter->afu_list_lock);
267 for (i = 0; i < adapter->slices; i++) {
268 if ((afu = adapter->afu[i])) {
269 pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
270 pci_channel_io_frozen);
271 cxl_context_detach_all(afu);
272 }
273 }
274
275 rc = cxl_h_reset_adapter(adapter->guest->handle);
276 for (i = 0; i < adapter->slices; i++) {
277 if (!rc && (afu = adapter->afu[i])) {
278 pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,
279 pci_channel_io_normal);
280 pci_error_handlers(afu, CXL_RESUME_EVENT, 0);
281 }
282 }
283 spin_unlock(&adapter->afu_list_lock);
284 return rc;
285}
286
287static int guest_alloc_one_irq(struct cxl *adapter)
288{
289 int irq;
290
291 spin_lock(&adapter->guest->irq_alloc_lock);
292 if (irq_alloc_range(adapter, 1, &irq))
293 irq = -ENOSPC;
294 spin_unlock(&adapter->guest->irq_alloc_lock);
295 return irq;
296}
297
298static void guest_release_one_irq(struct cxl *adapter, int irq)
299{
300 spin_lock(&adapter->guest->irq_alloc_lock);
301 irq_free_range(adapter, irq, 1);
302 spin_unlock(&adapter->guest->irq_alloc_lock);
303}
304
305static int guest_alloc_irq_ranges(struct cxl_irq_ranges *irqs,
306 struct cxl *adapter, unsigned int num)
307{
308 int i, try, irq;
309
310 memset(irqs, 0, sizeof(struct cxl_irq_ranges));
311
312 spin_lock(&adapter->guest->irq_alloc_lock);
313 for (i = 0; i < CXL_IRQ_RANGES && num; i++) {
314 try = num;
315 while (try) {
316 if (irq_alloc_range(adapter, try, &irq) == 0)
317 break;
318 try /= 2;
319 }
320 if (!try)
321 goto error;
322 irqs->offset[i] = irq;
323 irqs->range[i] = try;
324 num -= try;
325 }
326 if (num)
327 goto error;
328 spin_unlock(&adapter->guest->irq_alloc_lock);
329 return 0;
330
331error:
332 for (i = 0; i < CXL_IRQ_RANGES; i++)
333 irq_free_range(adapter, irqs->offset[i], irqs->range[i]);
334 spin_unlock(&adapter->guest->irq_alloc_lock);
335 return -ENOSPC;
336}
337
338static void guest_release_irq_ranges(struct cxl_irq_ranges *irqs,
339 struct cxl *adapter)
340{
341 int i;
342
343 spin_lock(&adapter->guest->irq_alloc_lock);
344 for (i = 0; i < CXL_IRQ_RANGES; i++)
345 irq_free_range(adapter, irqs->offset[i], irqs->range[i]);
346 spin_unlock(&adapter->guest->irq_alloc_lock);
347}
348
349static int guest_register_serr_irq(struct cxl_afu *afu)
350{
351 afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
352 dev_name(&afu->dev));
353 if (!afu->err_irq_name)
354 return -ENOMEM;
355
356 if (!(afu->serr_virq = cxl_map_irq(afu->adapter, afu->serr_hwirq,
357 guest_slice_irq_err, afu, afu->err_irq_name))) {
358 kfree(afu->err_irq_name);
359 afu->err_irq_name = NULL;
360 return -ENOMEM;
361 }
362
363 return 0;
364}
365
366static void guest_release_serr_irq(struct cxl_afu *afu)
367{
368 cxl_unmap_irq(afu->serr_virq, afu);
369 cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
370 kfree(afu->err_irq_name);
371}
372
373static int guest_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
374{
375 return cxl_h_control_faults(ctx->afu->guest->handle, ctx->process_token,
376 tfc >> 32, (psl_reset_mask != 0));
377}
378
379static void disable_afu_irqs(struct cxl_context *ctx)
380{
381 irq_hw_number_t hwirq;
382 unsigned int virq;
383 int r, i;
384
385 pr_devel("Disabling AFU(%d) interrupts\n", ctx->afu->slice);
386 for (r = 0; r < CXL_IRQ_RANGES; r++) {
387 hwirq = ctx->irqs.offset[r];
388 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
389 virq = irq_find_mapping(NULL, hwirq);
390 disable_irq(virq);
391 }
392 }
393}
394
395static void enable_afu_irqs(struct cxl_context *ctx)
396{
397 irq_hw_number_t hwirq;
398 unsigned int virq;
399 int r, i;
400
401 pr_devel("Enabling AFU(%d) interrupts\n", ctx->afu->slice);
402 for (r = 0; r < CXL_IRQ_RANGES; r++) {
403 hwirq = ctx->irqs.offset[r];
404 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
405 virq = irq_find_mapping(NULL, hwirq);
406 enable_irq(virq);
407 }
408 }
409}
410
411static int _guest_afu_cr_readXX(int sz, struct cxl_afu *afu, int cr_idx,
412 u64 offset, u64 *val)
413{
414 unsigned long cr;
415 char c;
416 int rc = 0;
417
418 if (afu->crs_len < sz)
419 return -ENOENT;
420
421 if (unlikely(offset >= afu->crs_len))
422 return -ERANGE;
423
424 cr = get_zeroed_page(GFP_KERNEL);
425 if (!cr)
426 return -ENOMEM;
427
428 rc = cxl_h_get_config(afu->guest->handle, cr_idx, offset,
429 virt_to_phys((void *)cr), sz);
430 if (rc)
431 goto err;
432
433 switch (sz) {
434 case 1:
435 c = *((char *) cr);
436 *val = c;
437 break;
438 case 2:
439 *val = in_le16((u16 *)cr);
440 break;
441 case 4:
442 *val = in_le32((unsigned *)cr);
443 break;
444 case 8:
445 *val = in_le64((u64 *)cr);
446 break;
447 default:
448 WARN_ON(1);
449 }
450err:
451 free_page(cr);
452 return rc;
453}
454
455static int guest_afu_cr_read32(struct cxl_afu *afu, int cr_idx, u64 offset,
456 u32 *out)
457{
458 int rc;
459 u64 val;
460
461 rc = _guest_afu_cr_readXX(4, afu, cr_idx, offset, &val);
462 if (!rc)
463 *out = (u32) val;
464 return rc;
465}
466
467static int guest_afu_cr_read16(struct cxl_afu *afu, int cr_idx, u64 offset,
468 u16 *out)
469{
470 int rc;
471 u64 val;
472
473 rc = _guest_afu_cr_readXX(2, afu, cr_idx, offset, &val);
474 if (!rc)
475 *out = (u16) val;
476 return rc;
477}
478
479static int guest_afu_cr_read8(struct cxl_afu *afu, int cr_idx, u64 offset,
480 u8 *out)
481{
482 int rc;
483 u64 val;
484
485 rc = _guest_afu_cr_readXX(1, afu, cr_idx, offset, &val);
486 if (!rc)
487 *out = (u8) val;
488 return rc;
489}
490
491static int guest_afu_cr_read64(struct cxl_afu *afu, int cr_idx, u64 offset,
492 u64 *out)
493{
494 return _guest_afu_cr_readXX(8, afu, cr_idx, offset, out);
495}
496
497static int guest_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
498{
499 /* config record is not writable from guest */
500 return -EPERM;
501}
502
503static int guest_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
504{
505 /* config record is not writable from guest */
506 return -EPERM;
507}
508
509static int guest_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
510{
511 /* config record is not writable from guest */
512 return -EPERM;
513}
514
515static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)
516{
517 struct cxl_process_element_hcall *elem;
518 struct cxl *adapter = ctx->afu->adapter;
519 const struct cred *cred;
520 u32 pid, idx;
521 int rc, r, i;
522 u64 mmio_addr, mmio_size;
523 __be64 flags = 0;
524
525 /* Must be 8 byte aligned and cannot cross a 4096 byte boundary */
526 if (!(elem = (struct cxl_process_element_hcall *)
527 get_zeroed_page(GFP_KERNEL)))
528 return -ENOMEM;
529
530 elem->version = cpu_to_be64(CXL_PROCESS_ELEMENT_VERSION);
531 if (ctx->kernel) {
532 pid = 0;
533 flags |= CXL_PE_TRANSLATION_ENABLED;
534 flags |= CXL_PE_PRIVILEGED_PROCESS;
535 if (mfmsr() & MSR_SF)
536 flags |= CXL_PE_64_BIT;
537 } else {
538 pid = current->pid;
539 flags |= CXL_PE_PROBLEM_STATE;
540 flags |= CXL_PE_TRANSLATION_ENABLED;
541 if (!test_tsk_thread_flag(current, TIF_32BIT))
542 flags |= CXL_PE_64_BIT;
543 cred = get_current_cred();
544 if (uid_eq(cred->euid, GLOBAL_ROOT_UID))
545 flags |= CXL_PE_PRIVILEGED_PROCESS;
546 put_cred(cred);
547 }
548 elem->flags = cpu_to_be64(flags);
549 elem->common.tid = cpu_to_be32(0); /* Unused */
550 elem->common.pid = cpu_to_be32(pid);
551 elem->common.csrp = cpu_to_be64(0); /* disable */
552 elem->common.u.psl8.aurp0 = cpu_to_be64(0); /* disable */
553 elem->common.u.psl8.aurp1 = cpu_to_be64(0); /* disable */
554
555 cxl_prefault(ctx, wed);
556
557 elem->common.u.psl8.sstp0 = cpu_to_be64(ctx->sstp0);
558 elem->common.u.psl8.sstp1 = cpu_to_be64(ctx->sstp1);
559
560 /*
561 * Ensure we have at least one interrupt allocated to take faults for
562 * kernel contexts that may not have allocated any AFU IRQs at all:
563 */
564 if (ctx->irqs.range[0] == 0) {
565 rc = afu_register_irqs(ctx, 0);
566 if (rc)
567 goto out_free;
568 }
569
570 for (r = 0; r < CXL_IRQ_RANGES; r++) {
571 for (i = 0; i < ctx->irqs.range[r]; i++) {
572 if (r == 0 && i == 0) {
573 elem->pslVirtualIsn = cpu_to_be32(ctx->irqs.offset[0]);
574 } else {
575 idx = ctx->irqs.offset[r] + i - adapter->guest->irq_base_offset;
576 elem->applicationVirtualIsnBitmap[idx / 8] |= 0x80 >> (idx % 8);
577 }
578 }
579 }
580 elem->common.amr = cpu_to_be64(amr);
581 elem->common.wed = cpu_to_be64(wed);
582
583 disable_afu_irqs(ctx);
584
585 rc = cxl_h_attach_process(ctx->afu->guest->handle, elem,
586 &ctx->process_token, &mmio_addr, &mmio_size);
587 if (rc == H_SUCCESS) {
588 if (ctx->master || !ctx->afu->pp_psa) {
589 ctx->psn_phys = ctx->afu->psn_phys;
590 ctx->psn_size = ctx->afu->adapter->ps_size;
591 } else {
592 ctx->psn_phys = mmio_addr;
593 ctx->psn_size = mmio_size;
594 }
595 if (ctx->afu->pp_psa && mmio_size &&
596 ctx->afu->pp_size == 0) {
597 /*
598 * There's no property in the device tree to read the
599 * pp_size. We only find out at the 1st attach.
600 * Compared to bare-metal, it is too late and we
601 * should really lock here. However, on powerVM,
602 * pp_size is really only used to display in /sys.
603 * Being discussed with pHyp for their next release.
604 */
605 ctx->afu->pp_size = mmio_size;
606 }
607 /* from PAPR: process element is bytes 4-7 of process token */
608 ctx->external_pe = ctx->process_token & 0xFFFFFFFF;
609 pr_devel("CXL pe=%i is known as %i for pHyp, mmio_size=%#llx",
610 ctx->pe, ctx->external_pe, ctx->psn_size);
611 ctx->pe_inserted = true;
612 enable_afu_irqs(ctx);
613 }
614
615out_free:
616 free_page((u64)elem);
617 return rc;
618}
619
620static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)
621{
622 pr_devel("in %s\n", __func__);
623
624 ctx->kernel = kernel;
625 if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
626 return attach_afu_directed(ctx, wed, amr);
627
628 /* dedicated mode not supported on FW840 */
629
630 return -EINVAL;
631}
632
633static int detach_afu_directed(struct cxl_context *ctx)
634{
635 if (!ctx->pe_inserted)
636 return 0;
637 if (cxl_h_detach_process(ctx->afu->guest->handle, ctx->process_token))
638 return -1;
639 return 0;
640}
641
642static int guest_detach_process(struct cxl_context *ctx)
643{
644 pr_devel("in %s\n", __func__);
645 trace_cxl_detach(ctx);
646
647 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
648 return -EIO;
649
650 if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
651 return detach_afu_directed(ctx);
652
653 return -EINVAL;
654}
655
656static void guest_release_afu(struct device *dev)
657{
658 struct cxl_afu *afu = to_cxl_afu(dev);
659
660 pr_devel("%s\n", __func__);
661
662 idr_destroy(&afu->contexts_idr);
663
664 kfree(afu->guest);
665 kfree(afu);
666}
667
668ssize_t cxl_guest_read_afu_vpd(struct cxl_afu *afu, void *buf, size_t len)
669{
670 return guest_collect_vpd(NULL, afu, buf, len);
671}
672
673#define ERR_BUFF_MAX_COPY_SIZE PAGE_SIZE
674static ssize_t guest_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
675 loff_t off, size_t count)
676{
677 void *tbuf = NULL;
678 int rc = 0;
679
680 tbuf = (void *) get_zeroed_page(GFP_KERNEL);
681 if (!tbuf)
682 return -ENOMEM;
683
684 rc = cxl_h_get_afu_err(afu->guest->handle,
685 off & 0x7,
686 virt_to_phys(tbuf),
687 count);
688 if (rc)
689 goto err;
690
691 if (count > ERR_BUFF_MAX_COPY_SIZE)
692 count = ERR_BUFF_MAX_COPY_SIZE - (off & 0x7);
693 memcpy(buf, tbuf, count);
694err:
695 free_page((u64)tbuf);
696
697 return rc;
698}
699
700static int guest_afu_check_and_enable(struct cxl_afu *afu)
701{
702 return 0;
703}
704
705static bool guest_support_attributes(const char *attr_name,
706 enum cxl_attrs type)
707{
708 switch (type) {
709 case CXL_ADAPTER_ATTRS:
710 if ((strcmp(attr_name, "base_image") == 0) ||
711 (strcmp(attr_name, "load_image_on_perst") == 0) ||
712 (strcmp(attr_name, "perst_reloads_same_image") == 0) ||
713 (strcmp(attr_name, "image_loaded") == 0))
714 return false;
715 break;
716 case CXL_AFU_MASTER_ATTRS:
717 if ((strcmp(attr_name, "pp_mmio_off") == 0))
718 return false;
719 break;
720 case CXL_AFU_ATTRS:
721 break;
722 default:
723 break;
724 }
725
726 return true;
727}
728
729static int activate_afu_directed(struct cxl_afu *afu)
730{
731 int rc;
732
733 dev_info(&afu->dev, "Activating AFU(%d) directed mode\n", afu->slice);
734
735 afu->current_mode = CXL_MODE_DIRECTED;
736
737 afu->num_procs = afu->max_procs_virtualised;
738
739 if ((rc = cxl_chardev_m_afu_add(afu)))
740 return rc;
741
742 if ((rc = cxl_sysfs_afu_m_add(afu)))
743 goto err;
744
745 if ((rc = cxl_chardev_s_afu_add(afu)))
746 goto err1;
747
748 return 0;
749err1:
750 cxl_sysfs_afu_m_remove(afu);
751err:
752 cxl_chardev_afu_remove(afu);
753 return rc;
754}
755
756static int guest_afu_activate_mode(struct cxl_afu *afu, int mode)
757{
758 if (!mode)
759 return 0;
760 if (!(mode & afu->modes_supported))
761 return -EINVAL;
762
763 if (mode == CXL_MODE_DIRECTED)
764 return activate_afu_directed(afu);
765
766 if (mode == CXL_MODE_DEDICATED)
767 dev_err(&afu->dev, "Dedicated mode not supported\n");
768
769 return -EINVAL;
770}
771
772static int deactivate_afu_directed(struct cxl_afu *afu)
773{
774 dev_info(&afu->dev, "Deactivating AFU(%d) directed mode\n", afu->slice);
775
776 afu->current_mode = 0;
777 afu->num_procs = 0;
778
779 cxl_sysfs_afu_m_remove(afu);
780 cxl_chardev_afu_remove(afu);
781
782 cxl_ops->afu_reset(afu);
783
784 return 0;
785}
786
787static int guest_afu_deactivate_mode(struct cxl_afu *afu, int mode)
788{
789 if (!mode)
790 return 0;
791 if (!(mode & afu->modes_supported))
792 return -EINVAL;
793
794 if (mode == CXL_MODE_DIRECTED)
795 return deactivate_afu_directed(afu);
796 return 0;
797}
798
799static int guest_afu_reset(struct cxl_afu *afu)
800{
801 pr_devel("AFU(%d) reset request\n", afu->slice);
802 return cxl_h_reset_afu(afu->guest->handle);
803}
804
805static int guest_map_slice_regs(struct cxl_afu *afu)
806{
807 if (!(afu->p2n_mmio = ioremap(afu->guest->p2n_phys, afu->guest->p2n_size))) {
808 dev_err(&afu->dev, "Error mapping AFU(%d) MMIO regions\n",
809 afu->slice);
810 return -ENOMEM;
811 }
812 return 0;
813}
814
815static void guest_unmap_slice_regs(struct cxl_afu *afu)
816{
817 if (afu->p2n_mmio)
818 iounmap(afu->p2n_mmio);
819}
820
821static int afu_update_state(struct cxl_afu *afu)
822{
823 int rc, cur_state;
824
825 rc = afu_read_error_state(afu, &cur_state);
826 if (rc)
827 return rc;
828
829 if (afu->guest->previous_state == cur_state)
830 return 0;
831
832 pr_devel("AFU(%d) update state to %#x\n", afu->slice, cur_state);
833
834 switch (cur_state) {
835 case H_STATE_NORMAL:
836 afu->guest->previous_state = cur_state;
837 break;
838
839 case H_STATE_DISABLE:
840 pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
841 pci_channel_io_frozen);
842
843 cxl_context_detach_all(afu);
844 if ((rc = cxl_ops->afu_reset(afu)))
845 pr_devel("reset hcall failed %d\n", rc);
846
847 rc = afu_read_error_state(afu, &cur_state);
848 if (!rc && cur_state == H_STATE_NORMAL) {
849 pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,
850 pci_channel_io_normal);
851 pci_error_handlers(afu, CXL_RESUME_EVENT, 0);
852 }
853 afu->guest->previous_state = 0;
854 break;
855
856 case H_STATE_TEMP_UNAVAILABLE:
857 afu->guest->previous_state = cur_state;
858 break;
859
860 case H_STATE_PERM_UNAVAILABLE:
861 dev_err(&afu->dev, "AFU is in permanent error state\n");
862 pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
863 pci_channel_io_perm_failure);
864 afu->guest->previous_state = cur_state;
865 break;
866
867 default:
868 pr_err("Unexpected AFU(%d) error state: %#x\n",
869 afu->slice, cur_state);
870 return -EINVAL;
871 }
872
873 return rc;
874}
875
876static void afu_handle_errstate(struct work_struct *work)
877{
878 struct cxl_afu_guest *afu_guest =
879 container_of(to_delayed_work(work), struct cxl_afu_guest, work_err);
880
881 if (!afu_update_state(afu_guest->parent) &&
882 afu_guest->previous_state == H_STATE_PERM_UNAVAILABLE)
883 return;
884
885 if (afu_guest->handle_err)
886 schedule_delayed_work(&afu_guest->work_err,
887 msecs_to_jiffies(3000));
888}
889
890static bool guest_link_ok(struct cxl *cxl, struct cxl_afu *afu)
891{
892 int state;
893
894 if (afu && (!afu_read_error_state(afu, &state))) {
895 if (state == H_STATE_NORMAL)
896 return true;
897 }
898
899 return false;
900}
901
902static int afu_properties_look_ok(struct cxl_afu *afu)
903{
904 if (afu->pp_irqs < 0) {
905 dev_err(&afu->dev, "Unexpected per-process minimum interrupt value\n");
906 return -EINVAL;
907 }
908
909 if (afu->max_procs_virtualised < 1) {
910 dev_err(&afu->dev, "Unexpected max number of processes virtualised value\n");
911 return -EINVAL;
912 }
913
914 return 0;
915}
916
917int cxl_guest_init_afu(struct cxl *adapter, int slice, struct device_node *afu_np)
918{
919 struct cxl_afu *afu;
920 bool free = true;
921 int rc;
922
923 pr_devel("in %s - AFU(%d)\n", __func__, slice);
924 if (!(afu = cxl_alloc_afu(adapter, slice)))
925 return -ENOMEM;
926
927 if (!(afu->guest = kzalloc(sizeof(struct cxl_afu_guest), GFP_KERNEL))) {
928 kfree(afu);
929 return -ENOMEM;
930 }
931
932 if ((rc = dev_set_name(&afu->dev, "afu%i.%i",
933 adapter->adapter_num,
934 slice)))
935 goto err1;
936
937 adapter->slices++;
938
939 if ((rc = cxl_of_read_afu_handle(afu, afu_np)))
940 goto err1;
941
942 if ((rc = cxl_ops->afu_reset(afu)))
943 goto err1;
944
945 if ((rc = cxl_of_read_afu_properties(afu, afu_np)))
946 goto err1;
947
948 if ((rc = afu_properties_look_ok(afu)))
949 goto err1;
950
951 if ((rc = guest_map_slice_regs(afu)))
952 goto err1;
953
954 if ((rc = guest_register_serr_irq(afu)))
955 goto err2;
956
957 /*
958 * After we call this function we must not free the afu directly, even
959 * if it returns an error!
960 */
961 if ((rc = cxl_register_afu(afu)))
962 goto err_put1;
963
964 if ((rc = cxl_sysfs_afu_add(afu)))
965 goto err_put1;
966
967 /*
968 * pHyp doesn't expose the programming models supported by the
969 * AFU. pHyp currently only supports directed mode. If it adds
970 * dedicated mode later, this version of cxl has no way to
971 * detect it. So we'll initialize the driver, but the first
972 * attach will fail.
973 * Being discussed with pHyp to do better (likely new property)
974 */
975 if (afu->max_procs_virtualised == 1)
976 afu->modes_supported = CXL_MODE_DEDICATED;
977 else
978 afu->modes_supported = CXL_MODE_DIRECTED;
979
980 if ((rc = cxl_afu_select_best_mode(afu)))
981 goto err_put2;
982
983 adapter->afu[afu->slice] = afu;
984
985 afu->enabled = true;
986
987 /*
988 * wake up the cpu periodically to check the state
989 * of the AFU using "afu" stored in the guest structure.
990 */
991 afu->guest->parent = afu;
992 afu->guest->handle_err = true;
993 INIT_DELAYED_WORK(&afu->guest->work_err, afu_handle_errstate);
994 schedule_delayed_work(&afu->guest->work_err, msecs_to_jiffies(1000));
995
996 if ((rc = cxl_pci_vphb_add(afu)))
997 dev_info(&afu->dev, "Can't register vPHB\n");
998
999 return 0;
1000
1001err_put2:
1002 cxl_sysfs_afu_remove(afu);
1003err_put1:
1004 device_unregister(&afu->dev);
1005 free = false;
1006 guest_release_serr_irq(afu);
1007err2:
1008 guest_unmap_slice_regs(afu);
1009err1:
1010 if (free) {
1011 kfree(afu->guest);
1012 kfree(afu);
1013 }
1014 return rc;
1015}
1016
1017void cxl_guest_remove_afu(struct cxl_afu *afu)
1018{
1019 if (!afu)
1020 return;
1021
1022 /* flush and stop pending job */
1023 afu->guest->handle_err = false;
1024 flush_delayed_work(&afu->guest->work_err);
1025
1026 cxl_pci_vphb_remove(afu);
1027 cxl_sysfs_afu_remove(afu);
1028
1029 spin_lock(&afu->adapter->afu_list_lock);
1030 afu->adapter->afu[afu->slice] = NULL;
1031 spin_unlock(&afu->adapter->afu_list_lock);
1032
1033 cxl_context_detach_all(afu);
1034 cxl_ops->afu_deactivate_mode(afu, afu->current_mode);
1035 guest_release_serr_irq(afu);
1036 guest_unmap_slice_regs(afu);
1037
1038 device_unregister(&afu->dev);
1039}
1040
1041static void free_adapter(struct cxl *adapter)
1042{
1043 struct irq_avail *cur;
1044 int i;
1045
1046 if (adapter->guest) {
1047 if (adapter->guest->irq_avail) {
1048 for (i = 0; i < adapter->guest->irq_nranges; i++) {
1049 cur = &adapter->guest->irq_avail[i];
1050 kfree(cur->bitmap);
1051 }
1052 kfree(adapter->guest->irq_avail);
1053 }
1054 kfree(adapter->guest->status);
1055 kfree(adapter->guest);
1056 }
1057 cxl_remove_adapter_nr(adapter);
1058 kfree(adapter);
1059}
1060
1061static int properties_look_ok(struct cxl *adapter)
1062{
1063 /* The absence of this property means that the operational
1064 * status is unknown or okay
1065 */
1066 if (strlen(adapter->guest->status) &&
1067 strcmp(adapter->guest->status, "okay")) {
1068 pr_err("ABORTING:Bad operational status of the device\n");
1069 return -EINVAL;
1070 }
1071
1072 return 0;
1073}
1074
1075ssize_t cxl_guest_read_adapter_vpd(struct cxl *adapter, void *buf, size_t len)
1076{
1077 return guest_collect_vpd(adapter, NULL, buf, len);
1078}
1079
1080void cxl_guest_remove_adapter(struct cxl *adapter)
1081{
1082 pr_devel("in %s\n", __func__);
1083
1084 cxl_sysfs_adapter_remove(adapter);
1085
1086 cxl_guest_remove_chardev(adapter);
1087 device_unregister(&adapter->dev);
1088}
1089
1090static void release_adapter(struct device *dev)
1091{
1092 free_adapter(to_cxl_adapter(dev));
1093}
1094
1095struct cxl *cxl_guest_init_adapter(struct device_node *np, struct platform_device *pdev)
1096{
1097 struct cxl *adapter;
1098 bool free = true;
1099 int rc;
1100
1101 if (!(adapter = cxl_alloc_adapter()))
1102 return ERR_PTR(-ENOMEM);
1103
1104 if (!(adapter->guest = kzalloc(sizeof(struct cxl_guest), GFP_KERNEL))) {
1105 free_adapter(adapter);
1106 return ERR_PTR(-ENOMEM);
1107 }
1108
1109 adapter->slices = 0;
1110 adapter->guest->pdev = pdev;
1111 adapter->dev.parent = &pdev->dev;
1112 adapter->dev.release = release_adapter;
1113 dev_set_drvdata(&pdev->dev, adapter);
1114
1115 /*
1116 * Hypervisor controls PSL timebase initialization (p1 register).
1117 * On FW840, PSL is initialized.
1118 */
1119 adapter->psl_timebase_synced = true;
1120
1121 if ((rc = cxl_of_read_adapter_handle(adapter, np)))
1122 goto err1;
1123
1124 if ((rc = cxl_of_read_adapter_properties(adapter, np)))
1125 goto err1;
1126
1127 if ((rc = properties_look_ok(adapter)))
1128 goto err1;
1129
1130 if ((rc = cxl_guest_add_chardev(adapter)))
1131 goto err1;
1132
1133 /*
1134 * After we call this function we must not free the adapter directly,
1135 * even if it returns an error!
1136 */
1137 if ((rc = cxl_register_adapter(adapter)))
1138 goto err_put1;
1139
1140 if ((rc = cxl_sysfs_adapter_add(adapter)))
1141 goto err_put1;
1142
1143 /* release the context lock as the adapter is configured */
1144 cxl_adapter_context_unlock(adapter);
1145
1146 return adapter;
1147
1148err_put1:
1149 device_unregister(&adapter->dev);
1150 free = false;
1151 cxl_guest_remove_chardev(adapter);
1152err1:
1153 if (free)
1154 free_adapter(adapter);
1155 return ERR_PTR(rc);
1156}
1157
1158void cxl_guest_reload_module(struct cxl *adapter)
1159{
1160 struct platform_device *pdev;
1161
1162 pdev = adapter->guest->pdev;
1163 cxl_guest_remove_adapter(adapter);
1164
1165 cxl_of_probe(pdev);
1166}
1167
1168const struct cxl_backend_ops cxl_guest_ops = {
1169 .module = THIS_MODULE,
1170 .adapter_reset = guest_reset,
1171 .alloc_one_irq = guest_alloc_one_irq,
1172 .release_one_irq = guest_release_one_irq,
1173 .alloc_irq_ranges = guest_alloc_irq_ranges,
1174 .release_irq_ranges = guest_release_irq_ranges,
1175 .setup_irq = NULL,
1176 .handle_psl_slice_error = guest_handle_psl_slice_error,
1177 .psl_interrupt = guest_psl_irq,
1178 .ack_irq = guest_ack_irq,
1179 .attach_process = guest_attach_process,
1180 .detach_process = guest_detach_process,
1181 .update_ivtes = NULL,
1182 .support_attributes = guest_support_attributes,
1183 .link_ok = guest_link_ok,
1184 .release_afu = guest_release_afu,
1185 .afu_read_err_buffer = guest_afu_read_err_buffer,
1186 .afu_check_and_enable = guest_afu_check_and_enable,
1187 .afu_activate_mode = guest_afu_activate_mode,
1188 .afu_deactivate_mode = guest_afu_deactivate_mode,
1189 .afu_reset = guest_afu_reset,
1190 .afu_cr_read8 = guest_afu_cr_read8,
1191 .afu_cr_read16 = guest_afu_cr_read16,
1192 .afu_cr_read32 = guest_afu_cr_read32,
1193 .afu_cr_read64 = guest_afu_cr_read64,
1194 .afu_cr_write8 = guest_afu_cr_write8,
1195 .afu_cr_write16 = guest_afu_cr_write16,
1196 .afu_cr_write32 = guest_afu_cr_write32,
1197 .read_adapter_vpd = cxl_guest_read_adapter_vpd,
1198};