Loading...
1// SPDX-License-Identifier: GPL-2.0+
2// Copyright 2017 IBM Corp.
3#include <linux/pci.h>
4#include <asm/pnv-ocxl.h>
5#include <misc/ocxl-config.h>
6#include "ocxl_internal.h"
7
8#define EXTRACT_BIT(val, bit) (!!(val & BIT(bit)))
9#define EXTRACT_BITS(val, s, e) ((val & GENMASK(e, s)) >> s)
10
11#define OCXL_DVSEC_AFU_IDX_MASK GENMASK(5, 0)
12#define OCXL_DVSEC_ACTAG_MASK GENMASK(11, 0)
13#define OCXL_DVSEC_PASID_MASK GENMASK(19, 0)
14#define OCXL_DVSEC_PASID_LOG_MASK GENMASK(4, 0)
15
16#define OCXL_DVSEC_TEMPL_VERSION 0x0
17#define OCXL_DVSEC_TEMPL_NAME 0x4
18#define OCXL_DVSEC_TEMPL_AFU_VERSION 0x1C
19#define OCXL_DVSEC_TEMPL_MMIO_GLOBAL 0x20
20#define OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ 0x28
21#define OCXL_DVSEC_TEMPL_MMIO_PP 0x30
22#define OCXL_DVSEC_TEMPL_MMIO_PP_SZ 0x38
23#define OCXL_DVSEC_TEMPL_ALL_MEM_SZ 0x3C
24#define OCXL_DVSEC_TEMPL_LPC_MEM_START 0x40
25#define OCXL_DVSEC_TEMPL_WWID 0x48
26#define OCXL_DVSEC_TEMPL_LPC_MEM_SZ 0x58
27
28#define OCXL_MAX_AFU_PER_FUNCTION 64
29#define OCXL_TEMPL_LEN_1_0 0x58
30#define OCXL_TEMPL_LEN_1_1 0x60
31#define OCXL_TEMPL_NAME_LEN 24
32#define OCXL_CFG_TIMEOUT 3
33
34static int find_dvsec(struct pci_dev *dev, int dvsec_id)
35{
36 return pci_find_dvsec_capability(dev, PCI_VENDOR_ID_IBM, dvsec_id);
37}
38
39static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
40{
41 int vsec = 0;
42 u16 vendor, id;
43 u8 idx;
44
45 while ((vsec = pci_find_next_ext_capability(dev, vsec,
46 OCXL_EXT_CAP_ID_DVSEC))) {
47 pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
48 &vendor);
49 pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
50
51 if (vendor == PCI_VENDOR_ID_IBM &&
52 id == OCXL_DVSEC_AFU_CTRL_ID) {
53 pci_read_config_byte(dev,
54 vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX,
55 &idx);
56 if (idx == afu_idx)
57 return vsec;
58 }
59 }
60 return 0;
61}
62
63/**
64 * get_function_0() - Find a related PCI device (function 0)
65 * @dev: PCI device to match
66 *
67 * Returns a pointer to the related device, or null if not found
68 */
69static struct pci_dev *get_function_0(struct pci_dev *dev)
70{
71 unsigned int devfn = PCI_DEVFN(PCI_SLOT(dev->devfn), 0);
72
73 return pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
74 dev->bus->number, devfn);
75}
76
77static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn)
78{
79 u16 val;
80 int pos;
81
82 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PASID);
83 if (!pos) {
84 /*
85 * PASID capability is not mandatory, but there
86 * shouldn't be any AFU
87 */
88 dev_dbg(&dev->dev, "Function doesn't require any PASID\n");
89 fn->max_pasid_log = -1;
90 goto out;
91 }
92 pci_read_config_word(dev, pos + PCI_PASID_CAP, &val);
93 fn->max_pasid_log = EXTRACT_BITS(val, 8, 12);
94
95out:
96 dev_dbg(&dev->dev, "PASID capability:\n");
97 dev_dbg(&dev->dev, " Max PASID log = %d\n", fn->max_pasid_log);
98}
99
100static int read_dvsec_tl(struct pci_dev *dev, struct ocxl_fn_config *fn)
101{
102 int pos;
103
104 pos = find_dvsec(dev, OCXL_DVSEC_TL_ID);
105 if (!pos && PCI_FUNC(dev->devfn) == 0) {
106 dev_err(&dev->dev, "Can't find TL DVSEC\n");
107 return -ENODEV;
108 }
109 if (pos && PCI_FUNC(dev->devfn) != 0) {
110 dev_err(&dev->dev, "TL DVSEC is only allowed on function 0\n");
111 return -ENODEV;
112 }
113 fn->dvsec_tl_pos = pos;
114 return 0;
115}
116
117static int read_dvsec_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
118{
119 int pos, afu_present;
120 u32 val;
121
122 pos = find_dvsec(dev, OCXL_DVSEC_FUNC_ID);
123 if (!pos) {
124 dev_err(&dev->dev, "Can't find function DVSEC\n");
125 return -ENODEV;
126 }
127 fn->dvsec_function_pos = pos;
128
129 pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val);
130 afu_present = EXTRACT_BIT(val, 31);
131 if (!afu_present) {
132 fn->max_afu_index = -1;
133 dev_dbg(&dev->dev, "Function doesn't define any AFU\n");
134 goto out;
135 }
136 fn->max_afu_index = EXTRACT_BITS(val, 24, 29);
137
138out:
139 dev_dbg(&dev->dev, "Function DVSEC:\n");
140 dev_dbg(&dev->dev, " Max AFU index = %d\n", fn->max_afu_index);
141 return 0;
142}
143
144static int read_dvsec_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn)
145{
146 int pos;
147
148 if (fn->max_afu_index < 0) {
149 fn->dvsec_afu_info_pos = -1;
150 return 0;
151 }
152
153 pos = find_dvsec(dev, OCXL_DVSEC_AFU_INFO_ID);
154 if (!pos) {
155 dev_err(&dev->dev, "Can't find AFU information DVSEC\n");
156 return -ENODEV;
157 }
158 fn->dvsec_afu_info_pos = pos;
159 return 0;
160}
161
162static int read_dvsec_vendor(struct pci_dev *dev)
163{
164 int pos;
165 u32 cfg, tlx, dlx, reset_reload;
166
167 /*
168 * vendor specific DVSEC, for IBM images only. Some older
169 * images may not have it
170 *
171 * It's only used on function 0 to specify the version of some
172 * logic blocks and to give access to special registers to
173 * enable host-based flashing.
174 */
175 if (PCI_FUNC(dev->devfn) != 0)
176 return 0;
177
178 pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID);
179 if (!pos)
180 return 0;
181
182 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_CFG_VERS, &cfg);
183 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_TLX_VERS, &tlx);
184 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_DLX_VERS, &dlx);
185 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
186 &reset_reload);
187
188 dev_dbg(&dev->dev, "Vendor specific DVSEC:\n");
189 dev_dbg(&dev->dev, " CFG version = 0x%x\n", cfg);
190 dev_dbg(&dev->dev, " TLX version = 0x%x\n", tlx);
191 dev_dbg(&dev->dev, " DLX version = 0x%x\n", dlx);
192 dev_dbg(&dev->dev, " ResetReload = 0x%x\n", reset_reload);
193 return 0;
194}
195
196/**
197 * get_dvsec_vendor0() - Find a related PCI device (function 0)
198 * @dev: PCI device to match
199 * @dev0: The PCI device (function 0) found
200 * @out_pos: The position of PCI device (function 0)
201 *
202 * Returns 0 on success, negative on failure.
203 *
204 * NOTE: If it's successful, the reference of dev0 is increased,
205 * so after using it, the callers must call pci_dev_put() to give
206 * up the reference.
207 */
208static int get_dvsec_vendor0(struct pci_dev *dev, struct pci_dev **dev0,
209 int *out_pos)
210{
211 int pos;
212
213 if (PCI_FUNC(dev->devfn) != 0) {
214 dev = get_function_0(dev);
215 if (!dev)
216 return -1;
217 } else {
218 dev = pci_dev_get(dev);
219 }
220 pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID);
221 if (!pos) {
222 pci_dev_put(dev);
223 return -1;
224 }
225 *dev0 = dev;
226 *out_pos = pos;
227 return 0;
228}
229
230int ocxl_config_get_reset_reload(struct pci_dev *dev, int *val)
231{
232 struct pci_dev *dev0;
233 u32 reset_reload;
234 int pos;
235
236 if (get_dvsec_vendor0(dev, &dev0, &pos))
237 return -1;
238
239 pci_read_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
240 &reset_reload);
241 pci_dev_put(dev0);
242 *val = !!(reset_reload & BIT(0));
243 return 0;
244}
245
246int ocxl_config_set_reset_reload(struct pci_dev *dev, int val)
247{
248 struct pci_dev *dev0;
249 u32 reset_reload;
250 int pos;
251
252 if (get_dvsec_vendor0(dev, &dev0, &pos))
253 return -1;
254
255 pci_read_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
256 &reset_reload);
257 if (val)
258 reset_reload |= BIT(0);
259 else
260 reset_reload &= ~BIT(0);
261 pci_write_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
262 reset_reload);
263 pci_dev_put(dev0);
264 return 0;
265}
266
267static int validate_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
268{
269 if (fn->max_pasid_log == -1 && fn->max_afu_index >= 0) {
270 dev_err(&dev->dev,
271 "AFUs are defined but no PASIDs are requested\n");
272 return -EINVAL;
273 }
274
275 if (fn->max_afu_index > OCXL_MAX_AFU_PER_FUNCTION) {
276 dev_err(&dev->dev,
277 "Max AFU index out of architectural limit (%d vs %d)\n",
278 fn->max_afu_index, OCXL_MAX_AFU_PER_FUNCTION);
279 return -EINVAL;
280 }
281 return 0;
282}
283
284int ocxl_config_read_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
285{
286 int rc;
287
288 read_pasid(dev, fn);
289
290 rc = read_dvsec_tl(dev, fn);
291 if (rc) {
292 dev_err(&dev->dev,
293 "Invalid Transaction Layer DVSEC configuration: %d\n",
294 rc);
295 return -ENODEV;
296 }
297
298 rc = read_dvsec_function(dev, fn);
299 if (rc) {
300 dev_err(&dev->dev,
301 "Invalid Function DVSEC configuration: %d\n", rc);
302 return -ENODEV;
303 }
304
305 rc = read_dvsec_afu_info(dev, fn);
306 if (rc) {
307 dev_err(&dev->dev, "Invalid AFU configuration: %d\n", rc);
308 return -ENODEV;
309 }
310
311 rc = read_dvsec_vendor(dev);
312 if (rc) {
313 dev_err(&dev->dev,
314 "Invalid vendor specific DVSEC configuration: %d\n",
315 rc);
316 return -ENODEV;
317 }
318
319 rc = validate_function(dev, fn);
320 return rc;
321}
322EXPORT_SYMBOL_GPL(ocxl_config_read_function);
323
324static int read_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn,
325 int offset, u32 *data)
326{
327 u32 val;
328 unsigned long timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT);
329 int pos = fn->dvsec_afu_info_pos;
330
331 /* Protect 'data valid' bit */
332 if (EXTRACT_BIT(offset, 31)) {
333 dev_err(&dev->dev, "Invalid offset in AFU info DVSEC\n");
334 return -EINVAL;
335 }
336
337 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, offset);
338 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, &val);
339 while (!EXTRACT_BIT(val, 31)) {
340 if (time_after_eq(jiffies, timeout)) {
341 dev_err(&dev->dev,
342 "Timeout while reading AFU info DVSEC (offset=%d)\n",
343 offset);
344 return -EBUSY;
345 }
346 cpu_relax();
347 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, &val);
348 }
349 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_DATA, data);
350 return 0;
351}
352
353/**
354 * read_template_version() - Read the template version from the AFU
355 * @dev: the device for the AFU
356 * @fn: the AFU offsets
357 * @len: outputs the template length
358 * @version: outputs the major<<8,minor version
359 *
360 * Returns 0 on success, negative on failure
361 */
362static int read_template_version(struct pci_dev *dev, struct ocxl_fn_config *fn,
363 u16 *len, u16 *version)
364{
365 u32 val32;
366 u8 major, minor;
367 int rc;
368
369 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_VERSION, &val32);
370 if (rc)
371 return rc;
372
373 *len = EXTRACT_BITS(val32, 16, 31);
374 major = EXTRACT_BITS(val32, 8, 15);
375 minor = EXTRACT_BITS(val32, 0, 7);
376 *version = (major << 8) + minor;
377 return 0;
378}
379
380int ocxl_config_check_afu_index(struct pci_dev *dev,
381 struct ocxl_fn_config *fn, int afu_idx)
382{
383 int rc;
384 u16 templ_version;
385 u16 len, expected_len;
386
387 pci_write_config_byte(dev,
388 fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX,
389 afu_idx);
390
391 rc = read_template_version(dev, fn, &len, &templ_version);
392 if (rc)
393 return rc;
394
395 /* AFU index map can have holes, in which case we read all 0's */
396 if (!templ_version && !len)
397 return 0;
398
399 dev_dbg(&dev->dev, "AFU descriptor template version %d.%d\n",
400 templ_version >> 8, templ_version & 0xFF);
401
402 switch (templ_version) {
403 case 0x0005: // v0.5 was used prior to the spec approval
404 case 0x0100:
405 expected_len = OCXL_TEMPL_LEN_1_0;
406 break;
407 case 0x0101:
408 expected_len = OCXL_TEMPL_LEN_1_1;
409 break;
410 default:
411 dev_warn(&dev->dev, "Unknown AFU template version %#x\n",
412 templ_version);
413 expected_len = len;
414 }
415 if (len != expected_len)
416 dev_warn(&dev->dev,
417 "Unexpected template length %#x in AFU information, expected %#x for version %#x\n",
418 len, expected_len, templ_version);
419 return 1;
420}
421
422static int read_afu_name(struct pci_dev *dev, struct ocxl_fn_config *fn,
423 struct ocxl_afu_config *afu)
424{
425 int i, rc;
426 u32 val, *ptr;
427
428 BUILD_BUG_ON(OCXL_AFU_NAME_SZ < OCXL_TEMPL_NAME_LEN);
429 for (i = 0; i < OCXL_TEMPL_NAME_LEN; i += 4) {
430 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_NAME + i, &val);
431 if (rc)
432 return rc;
433 ptr = (u32 *) &afu->name[i];
434 *ptr = le32_to_cpu((__force __le32) val);
435 }
436 afu->name[OCXL_AFU_NAME_SZ - 1] = '\0'; /* play safe */
437 return 0;
438}
439
440static int read_afu_mmio(struct pci_dev *dev, struct ocxl_fn_config *fn,
441 struct ocxl_afu_config *afu)
442{
443 int rc;
444 u32 val;
445
446 /*
447 * Global MMIO
448 */
449 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL, &val);
450 if (rc)
451 return rc;
452 afu->global_mmio_bar = EXTRACT_BITS(val, 0, 2);
453 afu->global_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16;
454
455 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL + 4, &val);
456 if (rc)
457 return rc;
458 afu->global_mmio_offset += (u64) val << 32;
459
460 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ, &val);
461 if (rc)
462 return rc;
463 afu->global_mmio_size = val;
464
465 /*
466 * Per-process MMIO
467 */
468 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP, &val);
469 if (rc)
470 return rc;
471 afu->pp_mmio_bar = EXTRACT_BITS(val, 0, 2);
472 afu->pp_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16;
473
474 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP + 4, &val);
475 if (rc)
476 return rc;
477 afu->pp_mmio_offset += (u64) val << 32;
478
479 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP_SZ, &val);
480 if (rc)
481 return rc;
482 afu->pp_mmio_stride = val;
483
484 return 0;
485}
486
487static int read_afu_control(struct pci_dev *dev, struct ocxl_afu_config *afu)
488{
489 int pos;
490 u8 val8;
491 u16 val16;
492
493 pos = find_dvsec_afu_ctrl(dev, afu->idx);
494 if (!pos) {
495 dev_err(&dev->dev, "Can't find AFU control DVSEC for AFU %d\n",
496 afu->idx);
497 return -ENODEV;
498 }
499 afu->dvsec_afu_control_pos = pos;
500
501 pci_read_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_SUP, &val8);
502 afu->pasid_supported_log = EXTRACT_BITS(val8, 0, 4);
503
504 pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, &val16);
505 afu->actag_supported = EXTRACT_BITS(val16, 0, 11);
506 return 0;
507}
508
509static bool char_allowed(int c)
510{
511 /*
512 * Permitted Characters : Alphanumeric, hyphen, underscore, comma
513 */
514 if ((c >= 0x30 && c <= 0x39) /* digits */ ||
515 (c >= 0x41 && c <= 0x5A) /* upper case */ ||
516 (c >= 0x61 && c <= 0x7A) /* lower case */ ||
517 c == 0 /* NULL */ ||
518 c == 0x2D /* - */ ||
519 c == 0x5F /* _ */ ||
520 c == 0x2C /* , */)
521 return true;
522 return false;
523}
524
525static int validate_afu(struct pci_dev *dev, struct ocxl_afu_config *afu)
526{
527 int i;
528
529 if (!afu->name[0]) {
530 dev_err(&dev->dev, "Empty AFU name\n");
531 return -EINVAL;
532 }
533 for (i = 0; i < OCXL_TEMPL_NAME_LEN; i++) {
534 if (!char_allowed(afu->name[i])) {
535 dev_err(&dev->dev,
536 "Invalid character in AFU name\n");
537 return -EINVAL;
538 }
539 }
540
541 if (afu->global_mmio_bar != 0 &&
542 afu->global_mmio_bar != 2 &&
543 afu->global_mmio_bar != 4) {
544 dev_err(&dev->dev, "Invalid global MMIO bar number\n");
545 return -EINVAL;
546 }
547 if (afu->pp_mmio_bar != 0 &&
548 afu->pp_mmio_bar != 2 &&
549 afu->pp_mmio_bar != 4) {
550 dev_err(&dev->dev, "Invalid per-process MMIO bar number\n");
551 return -EINVAL;
552 }
553 return 0;
554}
555
556/**
557 * read_afu_lpc_memory_info() - Populate AFU metadata regarding LPC memory
558 * @dev: the device for the AFU
559 * @fn: the AFU offsets
560 * @afu: the AFU struct to populate the LPC metadata into
561 *
562 * Returns 0 on success, negative on failure
563 */
564static int read_afu_lpc_memory_info(struct pci_dev *dev,
565 struct ocxl_fn_config *fn,
566 struct ocxl_afu_config *afu)
567{
568 int rc;
569 u32 val32;
570 u16 templ_version;
571 u16 templ_len;
572 u64 total_mem_size = 0;
573 u64 lpc_mem_size = 0;
574
575 afu->lpc_mem_offset = 0;
576 afu->lpc_mem_size = 0;
577 afu->special_purpose_mem_offset = 0;
578 afu->special_purpose_mem_size = 0;
579 /*
580 * For AFUs following template v1.0, the LPC memory covers the
581 * total memory. Its size is a power of 2.
582 *
583 * For AFUs with template >= v1.01, the total memory size is
584 * still a power of 2, but it is split in 2 parts:
585 * - the LPC memory, whose size can now be anything
586 * - the remainder memory is a special purpose memory, whose
587 * definition is AFU-dependent. It is not accessible through
588 * the usual commands for LPC memory
589 */
590 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_ALL_MEM_SZ, &val32);
591 if (rc)
592 return rc;
593
594 val32 = EXTRACT_BITS(val32, 0, 7);
595 if (!val32)
596 return 0; /* No LPC memory */
597
598 /*
599 * The configuration space spec allows for a memory size of up
600 * to 2^255 bytes.
601 *
602 * Current generation hardware uses 56-bit physical addresses,
603 * but we won't be able to get near close to that, as we won't
604 * have a hole big enough in the memory map. Let it pass in
605 * the driver for now. We'll get an error from the firmware
606 * when trying to configure something too big.
607 */
608 total_mem_size = 1ull << val32;
609
610 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_LPC_MEM_START, &val32);
611 if (rc)
612 return rc;
613
614 afu->lpc_mem_offset = val32;
615
616 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_LPC_MEM_START + 4, &val32);
617 if (rc)
618 return rc;
619
620 afu->lpc_mem_offset |= (u64) val32 << 32;
621
622 rc = read_template_version(dev, fn, &templ_len, &templ_version);
623 if (rc)
624 return rc;
625
626 if (templ_version >= 0x0101) {
627 rc = read_afu_info(dev, fn,
628 OCXL_DVSEC_TEMPL_LPC_MEM_SZ, &val32);
629 if (rc)
630 return rc;
631 lpc_mem_size = val32;
632
633 rc = read_afu_info(dev, fn,
634 OCXL_DVSEC_TEMPL_LPC_MEM_SZ + 4, &val32);
635 if (rc)
636 return rc;
637 lpc_mem_size |= (u64) val32 << 32;
638 } else {
639 lpc_mem_size = total_mem_size;
640 }
641 afu->lpc_mem_size = lpc_mem_size;
642
643 if (lpc_mem_size < total_mem_size) {
644 afu->special_purpose_mem_offset =
645 afu->lpc_mem_offset + lpc_mem_size;
646 afu->special_purpose_mem_size =
647 total_mem_size - lpc_mem_size;
648 }
649 return 0;
650}
651
652int ocxl_config_read_afu(struct pci_dev *dev, struct ocxl_fn_config *fn,
653 struct ocxl_afu_config *afu, u8 afu_idx)
654{
655 int rc;
656 u32 val32;
657
658 /*
659 * First, we need to write the AFU idx for the AFU we want to
660 * access.
661 */
662 WARN_ON((afu_idx & OCXL_DVSEC_AFU_IDX_MASK) != afu_idx);
663 afu->idx = afu_idx;
664 pci_write_config_byte(dev,
665 fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX,
666 afu->idx);
667
668 rc = read_afu_name(dev, fn, afu);
669 if (rc)
670 return rc;
671
672 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_AFU_VERSION, &val32);
673 if (rc)
674 return rc;
675 afu->version_major = EXTRACT_BITS(val32, 24, 31);
676 afu->version_minor = EXTRACT_BITS(val32, 16, 23);
677 afu->afuc_type = EXTRACT_BITS(val32, 14, 15);
678 afu->afum_type = EXTRACT_BITS(val32, 12, 13);
679 afu->profile = EXTRACT_BITS(val32, 0, 7);
680
681 rc = read_afu_mmio(dev, fn, afu);
682 if (rc)
683 return rc;
684
685 rc = read_afu_lpc_memory_info(dev, fn, afu);
686 if (rc)
687 return rc;
688
689 rc = read_afu_control(dev, afu);
690 if (rc)
691 return rc;
692
693 dev_dbg(&dev->dev, "AFU configuration:\n");
694 dev_dbg(&dev->dev, " name = %s\n", afu->name);
695 dev_dbg(&dev->dev, " version = %d.%d\n", afu->version_major,
696 afu->version_minor);
697 dev_dbg(&dev->dev, " global mmio bar = %hhu\n", afu->global_mmio_bar);
698 dev_dbg(&dev->dev, " global mmio offset = %#llx\n",
699 afu->global_mmio_offset);
700 dev_dbg(&dev->dev, " global mmio size = %#x\n", afu->global_mmio_size);
701 dev_dbg(&dev->dev, " pp mmio bar = %hhu\n", afu->pp_mmio_bar);
702 dev_dbg(&dev->dev, " pp mmio offset = %#llx\n", afu->pp_mmio_offset);
703 dev_dbg(&dev->dev, " pp mmio stride = %#x\n", afu->pp_mmio_stride);
704 dev_dbg(&dev->dev, " lpc_mem offset = %#llx\n", afu->lpc_mem_offset);
705 dev_dbg(&dev->dev, " lpc_mem size = %#llx\n", afu->lpc_mem_size);
706 dev_dbg(&dev->dev, " special purpose mem offset = %#llx\n",
707 afu->special_purpose_mem_offset);
708 dev_dbg(&dev->dev, " special purpose mem size = %#llx\n",
709 afu->special_purpose_mem_size);
710 dev_dbg(&dev->dev, " pasid supported (log) = %u\n",
711 afu->pasid_supported_log);
712 dev_dbg(&dev->dev, " actag supported = %u\n",
713 afu->actag_supported);
714
715 rc = validate_afu(dev, afu);
716 return rc;
717}
718EXPORT_SYMBOL_GPL(ocxl_config_read_afu);
719
720int ocxl_config_get_actag_info(struct pci_dev *dev, u16 *base, u16 *enabled,
721 u16 *supported)
722{
723 int rc;
724
725 /*
726 * This is really a simple wrapper for the kernel API, to
727 * avoid an external driver using ocxl as a library to call
728 * platform-dependent code
729 */
730 rc = pnv_ocxl_get_actag(dev, base, enabled, supported);
731 if (rc) {
732 dev_err(&dev->dev, "Can't get actag for device: %d\n", rc);
733 return rc;
734 }
735 return 0;
736}
737EXPORT_SYMBOL_GPL(ocxl_config_get_actag_info);
738
739void ocxl_config_set_afu_actag(struct pci_dev *dev, int pos, int actag_base,
740 int actag_count)
741{
742 u16 val;
743
744 val = actag_count & OCXL_DVSEC_ACTAG_MASK;
745 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_EN, val);
746
747 val = actag_base & OCXL_DVSEC_ACTAG_MASK;
748 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_BASE, val);
749}
750EXPORT_SYMBOL_GPL(ocxl_config_set_afu_actag);
751
752int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count)
753{
754 return pnv_ocxl_get_pasid_count(dev, count);
755}
756
757void ocxl_config_set_afu_pasid(struct pci_dev *dev, int pos, int pasid_base,
758 u32 pasid_count_log)
759{
760 u8 val8;
761 u32 val32;
762
763 val8 = pasid_count_log & OCXL_DVSEC_PASID_LOG_MASK;
764 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_EN, val8);
765
766 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE,
767 &val32);
768 val32 &= ~OCXL_DVSEC_PASID_MASK;
769 val32 |= pasid_base & OCXL_DVSEC_PASID_MASK;
770 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE,
771 val32);
772}
773EXPORT_SYMBOL_GPL(ocxl_config_set_afu_pasid);
774
775void ocxl_config_set_afu_state(struct pci_dev *dev, int pos, int enable)
776{
777 u8 val;
778
779 pci_read_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ENABLE, &val);
780 if (enable)
781 val |= 1;
782 else
783 val &= 0xFE;
784 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ENABLE, val);
785}
786EXPORT_SYMBOL_GPL(ocxl_config_set_afu_state);
787
788int ocxl_config_set_TL(struct pci_dev *dev, int tl_dvsec)
789{
790 u32 val;
791 __be32 *be32ptr;
792 u8 timers;
793 int i, rc;
794 long recv_cap;
795 char *recv_rate;
796
797 /*
798 * Skip on function != 0, as the TL can only be defined on 0
799 */
800 if (PCI_FUNC(dev->devfn) != 0)
801 return 0;
802
803 recv_rate = kzalloc(PNV_OCXL_TL_RATE_BUF_SIZE, GFP_KERNEL);
804 if (!recv_rate)
805 return -ENOMEM;
806 /*
807 * The spec defines 64 templates for messages in the
808 * Transaction Layer (TL).
809 *
810 * The host and device each support a subset, so we need to
811 * configure the transmitters on each side to send only
812 * templates the receiver understands, at a rate the receiver
813 * can process. Per the spec, template 0 must be supported by
814 * everybody. That's the template which has been used by the
815 * host and device so far.
816 *
817 * The sending rate limit must be set before the template is
818 * enabled.
819 */
820
821 /*
822 * Device -> host
823 */
824 rc = pnv_ocxl_get_tl_cap(dev, &recv_cap, recv_rate,
825 PNV_OCXL_TL_RATE_BUF_SIZE);
826 if (rc)
827 goto out;
828
829 for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) {
830 be32ptr = (__be32 *) &recv_rate[i];
831 pci_write_config_dword(dev,
832 tl_dvsec + OCXL_DVSEC_TL_SEND_RATE + i,
833 be32_to_cpu(*be32ptr));
834 }
835 val = recv_cap >> 32;
836 pci_write_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_SEND_CAP, val);
837 val = recv_cap & GENMASK(31, 0);
838 pci_write_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_SEND_CAP + 4, val);
839
840 /*
841 * Host -> device
842 */
843 for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) {
844 pci_read_config_dword(dev,
845 tl_dvsec + OCXL_DVSEC_TL_RECV_RATE + i,
846 &val);
847 be32ptr = (__be32 *) &recv_rate[i];
848 *be32ptr = cpu_to_be32(val);
849 }
850 pci_read_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_RECV_CAP, &val);
851 recv_cap = (long) val << 32;
852 pci_read_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_RECV_CAP + 4, &val);
853 recv_cap |= val;
854
855 rc = pnv_ocxl_set_tl_conf(dev, recv_cap, __pa(recv_rate),
856 PNV_OCXL_TL_RATE_BUF_SIZE);
857 if (rc)
858 goto out;
859
860 /*
861 * Opencapi commands needing to be retried are classified per
862 * the TL in 2 groups: short and long commands.
863 *
864 * The short back off timer it not used for now. It will be
865 * for opencapi 4.0.
866 *
867 * The long back off timer is typically used when an AFU hits
868 * a page fault but the NPU is already processing one. So the
869 * AFU needs to wait before it can resubmit. Having a value
870 * too low doesn't break anything, but can generate extra
871 * traffic on the link.
872 * We set it to 1.6 us for now. It's shorter than, but in the
873 * same order of magnitude as the time spent to process a page
874 * fault.
875 */
876 timers = 0x2 << 4; /* long timer = 1.6 us */
877 pci_write_config_byte(dev, tl_dvsec + OCXL_DVSEC_TL_BACKOFF_TIMERS,
878 timers);
879
880 rc = 0;
881out:
882 kfree(recv_rate);
883 return rc;
884}
885EXPORT_SYMBOL_GPL(ocxl_config_set_TL);
886
887int ocxl_config_terminate_pasid(struct pci_dev *dev, int afu_control, int pasid)
888{
889 u32 val;
890 unsigned long timeout;
891
892 pci_read_config_dword(dev, afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
893 &val);
894 if (EXTRACT_BIT(val, 20)) {
895 dev_err(&dev->dev,
896 "Can't terminate PASID %#x, previous termination didn't complete\n",
897 pasid);
898 return -EBUSY;
899 }
900
901 val &= ~OCXL_DVSEC_PASID_MASK;
902 val |= pasid & OCXL_DVSEC_PASID_MASK;
903 val |= BIT(20);
904 pci_write_config_dword(dev,
905 afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
906 val);
907
908 timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT);
909 pci_read_config_dword(dev, afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
910 &val);
911 while (EXTRACT_BIT(val, 20)) {
912 if (time_after_eq(jiffies, timeout)) {
913 dev_err(&dev->dev,
914 "Timeout while waiting for AFU to terminate PASID %#x\n",
915 pasid);
916 return -EBUSY;
917 }
918 cpu_relax();
919 pci_read_config_dword(dev,
920 afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
921 &val);
922 }
923 return 0;
924}
925EXPORT_SYMBOL_GPL(ocxl_config_terminate_pasid);
926
927void ocxl_config_set_actag(struct pci_dev *dev, int func_dvsec, u32 tag_first,
928 u32 tag_count)
929{
930 u32 val;
931
932 val = (tag_first & OCXL_DVSEC_ACTAG_MASK) << 16;
933 val |= tag_count & OCXL_DVSEC_ACTAG_MASK;
934 pci_write_config_dword(dev, func_dvsec + OCXL_DVSEC_FUNC_OFF_ACTAG,
935 val);
936}
937EXPORT_SYMBOL_GPL(ocxl_config_set_actag);
1// SPDX-License-Identifier: GPL-2.0+
2// Copyright 2017 IBM Corp.
3#include <linux/pci.h>
4#include <asm/pnv-ocxl.h>
5#include <misc/ocxl-config.h>
6#include "ocxl_internal.h"
7
8#define EXTRACT_BIT(val, bit) (!!(val & BIT(bit)))
9#define EXTRACT_BITS(val, s, e) ((val & GENMASK(e, s)) >> s)
10
11#define OCXL_DVSEC_AFU_IDX_MASK GENMASK(5, 0)
12#define OCXL_DVSEC_ACTAG_MASK GENMASK(11, 0)
13#define OCXL_DVSEC_PASID_MASK GENMASK(19, 0)
14#define OCXL_DVSEC_PASID_LOG_MASK GENMASK(4, 0)
15
16#define OCXL_DVSEC_TEMPL_VERSION 0x0
17#define OCXL_DVSEC_TEMPL_NAME 0x4
18#define OCXL_DVSEC_TEMPL_AFU_VERSION 0x1C
19#define OCXL_DVSEC_TEMPL_MMIO_GLOBAL 0x20
20#define OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ 0x28
21#define OCXL_DVSEC_TEMPL_MMIO_PP 0x30
22#define OCXL_DVSEC_TEMPL_MMIO_PP_SZ 0x38
23#define OCXL_DVSEC_TEMPL_ALL_MEM_SZ 0x3C
24#define OCXL_DVSEC_TEMPL_LPC_MEM_START 0x40
25#define OCXL_DVSEC_TEMPL_WWID 0x48
26#define OCXL_DVSEC_TEMPL_LPC_MEM_SZ 0x58
27
28#define OCXL_MAX_AFU_PER_FUNCTION 64
29#define OCXL_TEMPL_LEN_1_0 0x58
30#define OCXL_TEMPL_LEN_1_1 0x60
31#define OCXL_TEMPL_NAME_LEN 24
32#define OCXL_CFG_TIMEOUT 3
33
34static int find_dvsec(struct pci_dev *dev, int dvsec_id)
35{
36 int vsec = 0;
37 u16 vendor, id;
38
39 while ((vsec = pci_find_next_ext_capability(dev, vsec,
40 OCXL_EXT_CAP_ID_DVSEC))) {
41 pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
42 &vendor);
43 pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
44 if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id)
45 return vsec;
46 }
47 return 0;
48}
49
50static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
51{
52 int vsec = 0;
53 u16 vendor, id;
54 u8 idx;
55
56 while ((vsec = pci_find_next_ext_capability(dev, vsec,
57 OCXL_EXT_CAP_ID_DVSEC))) {
58 pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
59 &vendor);
60 pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
61
62 if (vendor == PCI_VENDOR_ID_IBM &&
63 id == OCXL_DVSEC_AFU_CTRL_ID) {
64 pci_read_config_byte(dev,
65 vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX,
66 &idx);
67 if (idx == afu_idx)
68 return vsec;
69 }
70 }
71 return 0;
72}
73
74/**
75 * get_function_0() - Find a related PCI device (function 0)
76 * @device: PCI device to match
77 *
78 * Returns a pointer to the related device, or null if not found
79 */
80static struct pci_dev *get_function_0(struct pci_dev *dev)
81{
82 unsigned int devfn = PCI_DEVFN(PCI_SLOT(dev->devfn), 0);
83
84 return pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
85 dev->bus->number, devfn);
86}
87
88static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn)
89{
90 u16 val;
91 int pos;
92
93 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PASID);
94 if (!pos) {
95 /*
96 * PASID capability is not mandatory, but there
97 * shouldn't be any AFU
98 */
99 dev_dbg(&dev->dev, "Function doesn't require any PASID\n");
100 fn->max_pasid_log = -1;
101 goto out;
102 }
103 pci_read_config_word(dev, pos + PCI_PASID_CAP, &val);
104 fn->max_pasid_log = EXTRACT_BITS(val, 8, 12);
105
106out:
107 dev_dbg(&dev->dev, "PASID capability:\n");
108 dev_dbg(&dev->dev, " Max PASID log = %d\n", fn->max_pasid_log);
109}
110
111static int read_dvsec_tl(struct pci_dev *dev, struct ocxl_fn_config *fn)
112{
113 int pos;
114
115 pos = find_dvsec(dev, OCXL_DVSEC_TL_ID);
116 if (!pos && PCI_FUNC(dev->devfn) == 0) {
117 dev_err(&dev->dev, "Can't find TL DVSEC\n");
118 return -ENODEV;
119 }
120 if (pos && PCI_FUNC(dev->devfn) != 0) {
121 dev_err(&dev->dev, "TL DVSEC is only allowed on function 0\n");
122 return -ENODEV;
123 }
124 fn->dvsec_tl_pos = pos;
125 return 0;
126}
127
128static int read_dvsec_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
129{
130 int pos, afu_present;
131 u32 val;
132
133 pos = find_dvsec(dev, OCXL_DVSEC_FUNC_ID);
134 if (!pos) {
135 dev_err(&dev->dev, "Can't find function DVSEC\n");
136 return -ENODEV;
137 }
138 fn->dvsec_function_pos = pos;
139
140 pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val);
141 afu_present = EXTRACT_BIT(val, 31);
142 if (!afu_present) {
143 fn->max_afu_index = -1;
144 dev_dbg(&dev->dev, "Function doesn't define any AFU\n");
145 goto out;
146 }
147 fn->max_afu_index = EXTRACT_BITS(val, 24, 29);
148
149out:
150 dev_dbg(&dev->dev, "Function DVSEC:\n");
151 dev_dbg(&dev->dev, " Max AFU index = %d\n", fn->max_afu_index);
152 return 0;
153}
154
155static int read_dvsec_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn)
156{
157 int pos;
158
159 if (fn->max_afu_index < 0) {
160 fn->dvsec_afu_info_pos = -1;
161 return 0;
162 }
163
164 pos = find_dvsec(dev, OCXL_DVSEC_AFU_INFO_ID);
165 if (!pos) {
166 dev_err(&dev->dev, "Can't find AFU information DVSEC\n");
167 return -ENODEV;
168 }
169 fn->dvsec_afu_info_pos = pos;
170 return 0;
171}
172
173static int read_dvsec_vendor(struct pci_dev *dev)
174{
175 int pos;
176 u32 cfg, tlx, dlx, reset_reload;
177
178 /*
179 * vendor specific DVSEC, for IBM images only. Some older
180 * images may not have it
181 *
182 * It's only used on function 0 to specify the version of some
183 * logic blocks and to give access to special registers to
184 * enable host-based flashing.
185 */
186 if (PCI_FUNC(dev->devfn) != 0)
187 return 0;
188
189 pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID);
190 if (!pos)
191 return 0;
192
193 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_CFG_VERS, &cfg);
194 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_TLX_VERS, &tlx);
195 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_DLX_VERS, &dlx);
196 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
197 &reset_reload);
198
199 dev_dbg(&dev->dev, "Vendor specific DVSEC:\n");
200 dev_dbg(&dev->dev, " CFG version = 0x%x\n", cfg);
201 dev_dbg(&dev->dev, " TLX version = 0x%x\n", tlx);
202 dev_dbg(&dev->dev, " DLX version = 0x%x\n", dlx);
203 dev_dbg(&dev->dev, " ResetReload = 0x%x\n", reset_reload);
204 return 0;
205}
206
207static int get_dvsec_vendor0(struct pci_dev *dev, struct pci_dev **dev0,
208 int *out_pos)
209{
210 int pos;
211
212 if (PCI_FUNC(dev->devfn) != 0) {
213 dev = get_function_0(dev);
214 if (!dev)
215 return -1;
216 }
217 pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID);
218 if (!pos)
219 return -1;
220 *dev0 = dev;
221 *out_pos = pos;
222 return 0;
223}
224
225int ocxl_config_get_reset_reload(struct pci_dev *dev, int *val)
226{
227 struct pci_dev *dev0;
228 u32 reset_reload;
229 int pos;
230
231 if (get_dvsec_vendor0(dev, &dev0, &pos))
232 return -1;
233
234 pci_read_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
235 &reset_reload);
236 *val = !!(reset_reload & BIT(0));
237 return 0;
238}
239
240int ocxl_config_set_reset_reload(struct pci_dev *dev, int val)
241{
242 struct pci_dev *dev0;
243 u32 reset_reload;
244 int pos;
245
246 if (get_dvsec_vendor0(dev, &dev0, &pos))
247 return -1;
248
249 pci_read_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
250 &reset_reload);
251 if (val)
252 reset_reload |= BIT(0);
253 else
254 reset_reload &= ~BIT(0);
255 pci_write_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
256 reset_reload);
257 return 0;
258}
259
260static int validate_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
261{
262 if (fn->max_pasid_log == -1 && fn->max_afu_index >= 0) {
263 dev_err(&dev->dev,
264 "AFUs are defined but no PASIDs are requested\n");
265 return -EINVAL;
266 }
267
268 if (fn->max_afu_index > OCXL_MAX_AFU_PER_FUNCTION) {
269 dev_err(&dev->dev,
270 "Max AFU index out of architectural limit (%d vs %d)\n",
271 fn->max_afu_index, OCXL_MAX_AFU_PER_FUNCTION);
272 return -EINVAL;
273 }
274 return 0;
275}
276
277int ocxl_config_read_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
278{
279 int rc;
280
281 read_pasid(dev, fn);
282
283 rc = read_dvsec_tl(dev, fn);
284 if (rc) {
285 dev_err(&dev->dev,
286 "Invalid Transaction Layer DVSEC configuration: %d\n",
287 rc);
288 return -ENODEV;
289 }
290
291 rc = read_dvsec_function(dev, fn);
292 if (rc) {
293 dev_err(&dev->dev,
294 "Invalid Function DVSEC configuration: %d\n", rc);
295 return -ENODEV;
296 }
297
298 rc = read_dvsec_afu_info(dev, fn);
299 if (rc) {
300 dev_err(&dev->dev, "Invalid AFU configuration: %d\n", rc);
301 return -ENODEV;
302 }
303
304 rc = read_dvsec_vendor(dev);
305 if (rc) {
306 dev_err(&dev->dev,
307 "Invalid vendor specific DVSEC configuration: %d\n",
308 rc);
309 return -ENODEV;
310 }
311
312 rc = validate_function(dev, fn);
313 return rc;
314}
315EXPORT_SYMBOL_GPL(ocxl_config_read_function);
316
317static int read_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn,
318 int offset, u32 *data)
319{
320 u32 val;
321 unsigned long timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT);
322 int pos = fn->dvsec_afu_info_pos;
323
324 /* Protect 'data valid' bit */
325 if (EXTRACT_BIT(offset, 31)) {
326 dev_err(&dev->dev, "Invalid offset in AFU info DVSEC\n");
327 return -EINVAL;
328 }
329
330 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, offset);
331 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, &val);
332 while (!EXTRACT_BIT(val, 31)) {
333 if (time_after_eq(jiffies, timeout)) {
334 dev_err(&dev->dev,
335 "Timeout while reading AFU info DVSEC (offset=%d)\n",
336 offset);
337 return -EBUSY;
338 }
339 cpu_relax();
340 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, &val);
341 }
342 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_DATA, data);
343 return 0;
344}
345
346/**
347 * read_template_version() - Read the template version from the AFU
348 * @dev: the device for the AFU
349 * @fn: the AFU offsets
350 * @len: outputs the template length
351 * @version: outputs the major<<8,minor version
352 *
353 * Returns 0 on success, negative on failure
354 */
355static int read_template_version(struct pci_dev *dev, struct ocxl_fn_config *fn,
356 u16 *len, u16 *version)
357{
358 u32 val32;
359 u8 major, minor;
360 int rc;
361
362 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_VERSION, &val32);
363 if (rc)
364 return rc;
365
366 *len = EXTRACT_BITS(val32, 16, 31);
367 major = EXTRACT_BITS(val32, 8, 15);
368 minor = EXTRACT_BITS(val32, 0, 7);
369 *version = (major << 8) + minor;
370 return 0;
371}
372
373int ocxl_config_check_afu_index(struct pci_dev *dev,
374 struct ocxl_fn_config *fn, int afu_idx)
375{
376 int rc;
377 u16 templ_version;
378 u16 len, expected_len;
379
380 pci_write_config_byte(dev,
381 fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX,
382 afu_idx);
383
384 rc = read_template_version(dev, fn, &len, &templ_version);
385 if (rc)
386 return rc;
387
388 /* AFU index map can have holes, in which case we read all 0's */
389 if (!templ_version && !len)
390 return 0;
391
392 dev_dbg(&dev->dev, "AFU descriptor template version %d.%d\n",
393 templ_version >> 8, templ_version & 0xFF);
394
395 switch (templ_version) {
396 case 0x0005: // v0.5 was used prior to the spec approval
397 case 0x0100:
398 expected_len = OCXL_TEMPL_LEN_1_0;
399 break;
400 case 0x0101:
401 expected_len = OCXL_TEMPL_LEN_1_1;
402 break;
403 default:
404 dev_warn(&dev->dev, "Unknown AFU template version %#x\n",
405 templ_version);
406 expected_len = len;
407 }
408 if (len != expected_len)
409 dev_warn(&dev->dev,
410 "Unexpected template length %#x in AFU information, expected %#x for version %#x\n",
411 len, expected_len, templ_version);
412 return 1;
413}
414
415static int read_afu_name(struct pci_dev *dev, struct ocxl_fn_config *fn,
416 struct ocxl_afu_config *afu)
417{
418 int i, rc;
419 u32 val, *ptr;
420
421 BUILD_BUG_ON(OCXL_AFU_NAME_SZ < OCXL_TEMPL_NAME_LEN);
422 for (i = 0; i < OCXL_TEMPL_NAME_LEN; i += 4) {
423 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_NAME + i, &val);
424 if (rc)
425 return rc;
426 ptr = (u32 *) &afu->name[i];
427 *ptr = le32_to_cpu((__force __le32) val);
428 }
429 afu->name[OCXL_AFU_NAME_SZ - 1] = '\0'; /* play safe */
430 return 0;
431}
432
433static int read_afu_mmio(struct pci_dev *dev, struct ocxl_fn_config *fn,
434 struct ocxl_afu_config *afu)
435{
436 int rc;
437 u32 val;
438
439 /*
440 * Global MMIO
441 */
442 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL, &val);
443 if (rc)
444 return rc;
445 afu->global_mmio_bar = EXTRACT_BITS(val, 0, 2);
446 afu->global_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16;
447
448 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL + 4, &val);
449 if (rc)
450 return rc;
451 afu->global_mmio_offset += (u64) val << 32;
452
453 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ, &val);
454 if (rc)
455 return rc;
456 afu->global_mmio_size = val;
457
458 /*
459 * Per-process MMIO
460 */
461 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP, &val);
462 if (rc)
463 return rc;
464 afu->pp_mmio_bar = EXTRACT_BITS(val, 0, 2);
465 afu->pp_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16;
466
467 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP + 4, &val);
468 if (rc)
469 return rc;
470 afu->pp_mmio_offset += (u64) val << 32;
471
472 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP_SZ, &val);
473 if (rc)
474 return rc;
475 afu->pp_mmio_stride = val;
476
477 return 0;
478}
479
480static int read_afu_control(struct pci_dev *dev, struct ocxl_afu_config *afu)
481{
482 int pos;
483 u8 val8;
484 u16 val16;
485
486 pos = find_dvsec_afu_ctrl(dev, afu->idx);
487 if (!pos) {
488 dev_err(&dev->dev, "Can't find AFU control DVSEC for AFU %d\n",
489 afu->idx);
490 return -ENODEV;
491 }
492 afu->dvsec_afu_control_pos = pos;
493
494 pci_read_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_SUP, &val8);
495 afu->pasid_supported_log = EXTRACT_BITS(val8, 0, 4);
496
497 pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, &val16);
498 afu->actag_supported = EXTRACT_BITS(val16, 0, 11);
499 return 0;
500}
501
502static bool char_allowed(int c)
503{
504 /*
505 * Permitted Characters : Alphanumeric, hyphen, underscore, comma
506 */
507 if ((c >= 0x30 && c <= 0x39) /* digits */ ||
508 (c >= 0x41 && c <= 0x5A) /* upper case */ ||
509 (c >= 0x61 && c <= 0x7A) /* lower case */ ||
510 c == 0 /* NULL */ ||
511 c == 0x2D /* - */ ||
512 c == 0x5F /* _ */ ||
513 c == 0x2C /* , */)
514 return true;
515 return false;
516}
517
518static int validate_afu(struct pci_dev *dev, struct ocxl_afu_config *afu)
519{
520 int i;
521
522 if (!afu->name[0]) {
523 dev_err(&dev->dev, "Empty AFU name\n");
524 return -EINVAL;
525 }
526 for (i = 0; i < OCXL_TEMPL_NAME_LEN; i++) {
527 if (!char_allowed(afu->name[i])) {
528 dev_err(&dev->dev,
529 "Invalid character in AFU name\n");
530 return -EINVAL;
531 }
532 }
533
534 if (afu->global_mmio_bar != 0 &&
535 afu->global_mmio_bar != 2 &&
536 afu->global_mmio_bar != 4) {
537 dev_err(&dev->dev, "Invalid global MMIO bar number\n");
538 return -EINVAL;
539 }
540 if (afu->pp_mmio_bar != 0 &&
541 afu->pp_mmio_bar != 2 &&
542 afu->pp_mmio_bar != 4) {
543 dev_err(&dev->dev, "Invalid per-process MMIO bar number\n");
544 return -EINVAL;
545 }
546 return 0;
547}
548
549/**
550 * read_afu_lpc_memory_info() - Populate AFU metadata regarding LPC memory
551 * @dev: the device for the AFU
552 * @fn: the AFU offsets
553 * @afu: the AFU struct to populate the LPC metadata into
554 *
555 * Returns 0 on success, negative on failure
556 */
557static int read_afu_lpc_memory_info(struct pci_dev *dev,
558 struct ocxl_fn_config *fn,
559 struct ocxl_afu_config *afu)
560{
561 int rc;
562 u32 val32;
563 u16 templ_version;
564 u16 templ_len;
565 u64 total_mem_size = 0;
566 u64 lpc_mem_size = 0;
567
568 afu->lpc_mem_offset = 0;
569 afu->lpc_mem_size = 0;
570 afu->special_purpose_mem_offset = 0;
571 afu->special_purpose_mem_size = 0;
572 /*
573 * For AFUs following template v1.0, the LPC memory covers the
574 * total memory. Its size is a power of 2.
575 *
576 * For AFUs with template >= v1.01, the total memory size is
577 * still a power of 2, but it is split in 2 parts:
578 * - the LPC memory, whose size can now be anything
579 * - the remainder memory is a special purpose memory, whose
580 * definition is AFU-dependent. It is not accessible through
581 * the usual commands for LPC memory
582 */
583 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_ALL_MEM_SZ, &val32);
584 if (rc)
585 return rc;
586
587 val32 = EXTRACT_BITS(val32, 0, 7);
588 if (!val32)
589 return 0; /* No LPC memory */
590
591 /*
592 * The configuration space spec allows for a memory size of up
593 * to 2^255 bytes.
594 *
595 * Current generation hardware uses 56-bit physical addresses,
596 * but we won't be able to get near close to that, as we won't
597 * have a hole big enough in the memory map. Let it pass in
598 * the driver for now. We'll get an error from the firmware
599 * when trying to configure something too big.
600 */
601 total_mem_size = 1ull << val32;
602
603 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_LPC_MEM_START, &val32);
604 if (rc)
605 return rc;
606
607 afu->lpc_mem_offset = val32;
608
609 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_LPC_MEM_START + 4, &val32);
610 if (rc)
611 return rc;
612
613 afu->lpc_mem_offset |= (u64) val32 << 32;
614
615 rc = read_template_version(dev, fn, &templ_len, &templ_version);
616 if (rc)
617 return rc;
618
619 if (templ_version >= 0x0101) {
620 rc = read_afu_info(dev, fn,
621 OCXL_DVSEC_TEMPL_LPC_MEM_SZ, &val32);
622 if (rc)
623 return rc;
624 lpc_mem_size = val32;
625
626 rc = read_afu_info(dev, fn,
627 OCXL_DVSEC_TEMPL_LPC_MEM_SZ + 4, &val32);
628 if (rc)
629 return rc;
630 lpc_mem_size |= (u64) val32 << 32;
631 } else {
632 lpc_mem_size = total_mem_size;
633 }
634 afu->lpc_mem_size = lpc_mem_size;
635
636 if (lpc_mem_size < total_mem_size) {
637 afu->special_purpose_mem_offset =
638 afu->lpc_mem_offset + lpc_mem_size;
639 afu->special_purpose_mem_size =
640 total_mem_size - lpc_mem_size;
641 }
642 return 0;
643}
644
645int ocxl_config_read_afu(struct pci_dev *dev, struct ocxl_fn_config *fn,
646 struct ocxl_afu_config *afu, u8 afu_idx)
647{
648 int rc;
649 u32 val32;
650
651 /*
652 * First, we need to write the AFU idx for the AFU we want to
653 * access.
654 */
655 WARN_ON((afu_idx & OCXL_DVSEC_AFU_IDX_MASK) != afu_idx);
656 afu->idx = afu_idx;
657 pci_write_config_byte(dev,
658 fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX,
659 afu->idx);
660
661 rc = read_afu_name(dev, fn, afu);
662 if (rc)
663 return rc;
664
665 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_AFU_VERSION, &val32);
666 if (rc)
667 return rc;
668 afu->version_major = EXTRACT_BITS(val32, 24, 31);
669 afu->version_minor = EXTRACT_BITS(val32, 16, 23);
670 afu->afuc_type = EXTRACT_BITS(val32, 14, 15);
671 afu->afum_type = EXTRACT_BITS(val32, 12, 13);
672 afu->profile = EXTRACT_BITS(val32, 0, 7);
673
674 rc = read_afu_mmio(dev, fn, afu);
675 if (rc)
676 return rc;
677
678 rc = read_afu_lpc_memory_info(dev, fn, afu);
679 if (rc)
680 return rc;
681
682 rc = read_afu_control(dev, afu);
683 if (rc)
684 return rc;
685
686 dev_dbg(&dev->dev, "AFU configuration:\n");
687 dev_dbg(&dev->dev, " name = %s\n", afu->name);
688 dev_dbg(&dev->dev, " version = %d.%d\n", afu->version_major,
689 afu->version_minor);
690 dev_dbg(&dev->dev, " global mmio bar = %hhu\n", afu->global_mmio_bar);
691 dev_dbg(&dev->dev, " global mmio offset = %#llx\n",
692 afu->global_mmio_offset);
693 dev_dbg(&dev->dev, " global mmio size = %#x\n", afu->global_mmio_size);
694 dev_dbg(&dev->dev, " pp mmio bar = %hhu\n", afu->pp_mmio_bar);
695 dev_dbg(&dev->dev, " pp mmio offset = %#llx\n", afu->pp_mmio_offset);
696 dev_dbg(&dev->dev, " pp mmio stride = %#x\n", afu->pp_mmio_stride);
697 dev_dbg(&dev->dev, " lpc_mem offset = %#llx\n", afu->lpc_mem_offset);
698 dev_dbg(&dev->dev, " lpc_mem size = %#llx\n", afu->lpc_mem_size);
699 dev_dbg(&dev->dev, " special purpose mem offset = %#llx\n",
700 afu->special_purpose_mem_offset);
701 dev_dbg(&dev->dev, " special purpose mem size = %#llx\n",
702 afu->special_purpose_mem_size);
703 dev_dbg(&dev->dev, " pasid supported (log) = %u\n",
704 afu->pasid_supported_log);
705 dev_dbg(&dev->dev, " actag supported = %u\n",
706 afu->actag_supported);
707
708 rc = validate_afu(dev, afu);
709 return rc;
710}
711EXPORT_SYMBOL_GPL(ocxl_config_read_afu);
712
713int ocxl_config_get_actag_info(struct pci_dev *dev, u16 *base, u16 *enabled,
714 u16 *supported)
715{
716 int rc;
717
718 /*
719 * This is really a simple wrapper for the kernel API, to
720 * avoid an external driver using ocxl as a library to call
721 * platform-dependent code
722 */
723 rc = pnv_ocxl_get_actag(dev, base, enabled, supported);
724 if (rc) {
725 dev_err(&dev->dev, "Can't get actag for device: %d\n", rc);
726 return rc;
727 }
728 return 0;
729}
730EXPORT_SYMBOL_GPL(ocxl_config_get_actag_info);
731
732void ocxl_config_set_afu_actag(struct pci_dev *dev, int pos, int actag_base,
733 int actag_count)
734{
735 u16 val;
736
737 val = actag_count & OCXL_DVSEC_ACTAG_MASK;
738 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_EN, val);
739
740 val = actag_base & OCXL_DVSEC_ACTAG_MASK;
741 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_BASE, val);
742}
743EXPORT_SYMBOL_GPL(ocxl_config_set_afu_actag);
744
745int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count)
746{
747 return pnv_ocxl_get_pasid_count(dev, count);
748}
749
750void ocxl_config_set_afu_pasid(struct pci_dev *dev, int pos, int pasid_base,
751 u32 pasid_count_log)
752{
753 u8 val8;
754 u32 val32;
755
756 val8 = pasid_count_log & OCXL_DVSEC_PASID_LOG_MASK;
757 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_EN, val8);
758
759 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE,
760 &val32);
761 val32 &= ~OCXL_DVSEC_PASID_MASK;
762 val32 |= pasid_base & OCXL_DVSEC_PASID_MASK;
763 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE,
764 val32);
765}
766EXPORT_SYMBOL_GPL(ocxl_config_set_afu_pasid);
767
768void ocxl_config_set_afu_state(struct pci_dev *dev, int pos, int enable)
769{
770 u8 val;
771
772 pci_read_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ENABLE, &val);
773 if (enable)
774 val |= 1;
775 else
776 val &= 0xFE;
777 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ENABLE, val);
778}
779EXPORT_SYMBOL_GPL(ocxl_config_set_afu_state);
780
781int ocxl_config_set_TL(struct pci_dev *dev, int tl_dvsec)
782{
783 u32 val;
784 __be32 *be32ptr;
785 u8 timers;
786 int i, rc;
787 long recv_cap;
788 char *recv_rate;
789
790 /*
791 * Skip on function != 0, as the TL can only be defined on 0
792 */
793 if (PCI_FUNC(dev->devfn) != 0)
794 return 0;
795
796 recv_rate = kzalloc(PNV_OCXL_TL_RATE_BUF_SIZE, GFP_KERNEL);
797 if (!recv_rate)
798 return -ENOMEM;
799 /*
800 * The spec defines 64 templates for messages in the
801 * Transaction Layer (TL).
802 *
803 * The host and device each support a subset, so we need to
804 * configure the transmitters on each side to send only
805 * templates the receiver understands, at a rate the receiver
806 * can process. Per the spec, template 0 must be supported by
807 * everybody. That's the template which has been used by the
808 * host and device so far.
809 *
810 * The sending rate limit must be set before the template is
811 * enabled.
812 */
813
814 /*
815 * Device -> host
816 */
817 rc = pnv_ocxl_get_tl_cap(dev, &recv_cap, recv_rate,
818 PNV_OCXL_TL_RATE_BUF_SIZE);
819 if (rc)
820 goto out;
821
822 for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) {
823 be32ptr = (__be32 *) &recv_rate[i];
824 pci_write_config_dword(dev,
825 tl_dvsec + OCXL_DVSEC_TL_SEND_RATE + i,
826 be32_to_cpu(*be32ptr));
827 }
828 val = recv_cap >> 32;
829 pci_write_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_SEND_CAP, val);
830 val = recv_cap & GENMASK(31, 0);
831 pci_write_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_SEND_CAP + 4, val);
832
833 /*
834 * Host -> device
835 */
836 for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) {
837 pci_read_config_dword(dev,
838 tl_dvsec + OCXL_DVSEC_TL_RECV_RATE + i,
839 &val);
840 be32ptr = (__be32 *) &recv_rate[i];
841 *be32ptr = cpu_to_be32(val);
842 }
843 pci_read_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_RECV_CAP, &val);
844 recv_cap = (long) val << 32;
845 pci_read_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_RECV_CAP + 4, &val);
846 recv_cap |= val;
847
848 rc = pnv_ocxl_set_tl_conf(dev, recv_cap, __pa(recv_rate),
849 PNV_OCXL_TL_RATE_BUF_SIZE);
850 if (rc)
851 goto out;
852
853 /*
854 * Opencapi commands needing to be retried are classified per
855 * the TL in 2 groups: short and long commands.
856 *
857 * The short back off timer it not used for now. It will be
858 * for opencapi 4.0.
859 *
860 * The long back off timer is typically used when an AFU hits
861 * a page fault but the NPU is already processing one. So the
862 * AFU needs to wait before it can resubmit. Having a value
863 * too low doesn't break anything, but can generate extra
864 * traffic on the link.
865 * We set it to 1.6 us for now. It's shorter than, but in the
866 * same order of magnitude as the time spent to process a page
867 * fault.
868 */
869 timers = 0x2 << 4; /* long timer = 1.6 us */
870 pci_write_config_byte(dev, tl_dvsec + OCXL_DVSEC_TL_BACKOFF_TIMERS,
871 timers);
872
873 rc = 0;
874out:
875 kfree(recv_rate);
876 return rc;
877}
878EXPORT_SYMBOL_GPL(ocxl_config_set_TL);
879
880int ocxl_config_terminate_pasid(struct pci_dev *dev, int afu_control, int pasid)
881{
882 u32 val;
883 unsigned long timeout;
884
885 pci_read_config_dword(dev, afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
886 &val);
887 if (EXTRACT_BIT(val, 20)) {
888 dev_err(&dev->dev,
889 "Can't terminate PASID %#x, previous termination didn't complete\n",
890 pasid);
891 return -EBUSY;
892 }
893
894 val &= ~OCXL_DVSEC_PASID_MASK;
895 val |= pasid & OCXL_DVSEC_PASID_MASK;
896 val |= BIT(20);
897 pci_write_config_dword(dev,
898 afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
899 val);
900
901 timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT);
902 pci_read_config_dword(dev, afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
903 &val);
904 while (EXTRACT_BIT(val, 20)) {
905 if (time_after_eq(jiffies, timeout)) {
906 dev_err(&dev->dev,
907 "Timeout while waiting for AFU to terminate PASID %#x\n",
908 pasid);
909 return -EBUSY;
910 }
911 cpu_relax();
912 pci_read_config_dword(dev,
913 afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
914 &val);
915 }
916 return 0;
917}
918EXPORT_SYMBOL_GPL(ocxl_config_terminate_pasid);
919
920void ocxl_config_set_actag(struct pci_dev *dev, int func_dvsec, u32 tag_first,
921 u32 tag_count)
922{
923 u32 val;
924
925 val = (tag_first & OCXL_DVSEC_ACTAG_MASK) << 16;
926 val |= tag_count & OCXL_DVSEC_ACTAG_MASK;
927 pci_write_config_dword(dev, func_dvsec + OCXL_DVSEC_FUNC_OFF_ACTAG,
928 val);
929}
930EXPORT_SYMBOL_GPL(ocxl_config_set_actag);