Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright IBM Corp. 2020
4 *
5 * Author(s):
6 * Pierre Morel <pmorel@linux.ibm.com>
7 *
8 */
9
10#define KMSG_COMPONENT "zpci"
11#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
12
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/err.h>
16#include <linux/export.h>
17#include <linux/delay.h>
18#include <linux/seq_file.h>
19#include <linux/jump_label.h>
20#include <linux/pci.h>
21#include <linux/printk.h>
22
23#include <asm/pci_clp.h>
24#include <asm/pci_dma.h>
25
26#include "pci_bus.h"
27#include "pci_iov.h"
28
29static LIST_HEAD(zbus_list);
30static DEFINE_MUTEX(zbus_list_lock);
31static int zpci_nb_devices;
32
33/* zpci_bus_prepare_device - Prepare a zPCI function for scanning
34 * @zdev: the zPCI function to be prepared
35 *
36 * The PCI resources for the function are set up and added to its zbus and the
37 * function is enabled. The function must be added to a zbus which must have
38 * a PCI bus created. If an error occurs the zPCI function is not enabled.
39 *
40 * Return: 0 on success, an error code otherwise
41 */
42static int zpci_bus_prepare_device(struct zpci_dev *zdev)
43{
44 int rc, i;
45
46 if (!zdev_enabled(zdev)) {
47 rc = zpci_enable_device(zdev);
48 if (rc)
49 return rc;
50 }
51
52 if (!zdev->has_resources) {
53 zpci_setup_bus_resources(zdev);
54 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
55 if (zdev->bars[i].res)
56 pci_bus_add_resource(zdev->zbus->bus, zdev->bars[i].res);
57 }
58 }
59
60 return 0;
61}
62
63/* zpci_bus_scan_device - Scan a single device adding it to the PCI core
64 * @zdev: the zdev to be scanned
65 *
66 * Scans the PCI function making it available to the common PCI code.
67 *
68 * Return: 0 on success, an error value otherwise
69 */
70int zpci_bus_scan_device(struct zpci_dev *zdev)
71{
72 struct pci_dev *pdev;
73 int rc;
74
75 rc = zpci_bus_prepare_device(zdev);
76 if (rc)
77 return rc;
78
79 pdev = pci_scan_single_device(zdev->zbus->bus, zdev->devfn);
80 if (!pdev)
81 return -ENODEV;
82
83 pci_lock_rescan_remove();
84 pci_bus_add_device(pdev);
85 pci_unlock_rescan_remove();
86
87 return 0;
88}
89
90/* zpci_bus_remove_device - Removes the given zdev from the PCI core
91 * @zdev: the zdev to be removed from the PCI core
92 * @set_error: if true the device's error state is set to permanent failure
93 *
94 * Sets a zPCI device to a configured but offline state; the zPCI
95 * device is still accessible through its hotplug slot and the zPCI
96 * API but is removed from the common code PCI bus, making it
97 * no longer available to drivers.
98 */
99void zpci_bus_remove_device(struct zpci_dev *zdev, bool set_error)
100{
101 struct zpci_bus *zbus = zdev->zbus;
102 struct pci_dev *pdev;
103
104 if (!zdev->zbus->bus)
105 return;
106
107 pdev = pci_get_slot(zbus->bus, zdev->devfn);
108 if (pdev) {
109 if (set_error)
110 pdev->error_state = pci_channel_io_perm_failure;
111 if (pdev->is_virtfn) {
112 zpci_iov_remove_virtfn(pdev, zdev->vfn);
113 /* balance pci_get_slot */
114 pci_dev_put(pdev);
115 return;
116 }
117 pci_stop_and_remove_bus_device_locked(pdev);
118 /* balance pci_get_slot */
119 pci_dev_put(pdev);
120 }
121}
122
123/* zpci_bus_scan_bus - Scan all configured zPCI functions on the bus
124 * @zbus: the zbus to be scanned
125 *
126 * Enables and scans all PCI functions on the bus making them available to the
127 * common PCI code. If a PCI function fails to be initialized an error will be
128 * returned but attempts will still be made for all other functions on the bus.
129 *
130 * Return: 0 on success, an error value otherwise
131 */
132int zpci_bus_scan_bus(struct zpci_bus *zbus)
133{
134 struct zpci_dev *zdev;
135 int devfn, rc, ret = 0;
136
137 for (devfn = 0; devfn < ZPCI_FUNCTIONS_PER_BUS; devfn++) {
138 zdev = zbus->function[devfn];
139 if (zdev && zdev->state == ZPCI_FN_STATE_CONFIGURED) {
140 rc = zpci_bus_prepare_device(zdev);
141 if (rc)
142 ret = -EIO;
143 }
144 }
145
146 pci_lock_rescan_remove();
147 pci_scan_child_bus(zbus->bus);
148 pci_bus_add_devices(zbus->bus);
149 pci_unlock_rescan_remove();
150
151 return ret;
152}
153
154/* zpci_bus_scan_busses - Scan all registered busses
155 *
156 * Scan all available zbusses
157 *
158 */
159void zpci_bus_scan_busses(void)
160{
161 struct zpci_bus *zbus = NULL;
162
163 mutex_lock(&zbus_list_lock);
164 list_for_each_entry(zbus, &zbus_list, bus_next) {
165 zpci_bus_scan_bus(zbus);
166 cond_resched();
167 }
168 mutex_unlock(&zbus_list_lock);
169}
170
171static bool zpci_bus_is_multifunction_root(struct zpci_dev *zdev)
172{
173 return !s390_pci_no_rid && zdev->rid_available &&
174 !zdev->vfn;
175}
176
177/* zpci_bus_create_pci_bus - Create the PCI bus associated with this zbus
178 * @zbus: the zbus holding the zdevices
179 * @fr: PCI root function that will determine the bus's domain, and bus speed
180 * @ops: the pci operations
181 *
182 * The PCI function @fr determines the domain (its UID), multifunction property
183 * and maximum bus speed of the entire bus.
184 *
185 * Return: 0 on success, an error code otherwise
186 */
187static int zpci_bus_create_pci_bus(struct zpci_bus *zbus, struct zpci_dev *fr, struct pci_ops *ops)
188{
189 struct pci_bus *bus;
190 int domain;
191
192 domain = zpci_alloc_domain((u16)fr->uid);
193 if (domain < 0)
194 return domain;
195
196 zbus->domain_nr = domain;
197 zbus->multifunction = zpci_bus_is_multifunction_root(fr);
198 zbus->max_bus_speed = fr->max_bus_speed;
199
200 /*
201 * Note that the zbus->resources are taken over and zbus->resources
202 * is empty after a successful call
203 */
204 bus = pci_create_root_bus(NULL, ZPCI_BUS_NR, ops, zbus, &zbus->resources);
205 if (!bus) {
206 zpci_free_domain(zbus->domain_nr);
207 return -EFAULT;
208 }
209
210 zbus->bus = bus;
211
212 return 0;
213}
214
215static void zpci_bus_release(struct kref *kref)
216{
217 struct zpci_bus *zbus = container_of(kref, struct zpci_bus, kref);
218
219 if (zbus->bus) {
220 pci_lock_rescan_remove();
221 pci_stop_root_bus(zbus->bus);
222
223 zpci_free_domain(zbus->domain_nr);
224 pci_free_resource_list(&zbus->resources);
225
226 pci_remove_root_bus(zbus->bus);
227 pci_unlock_rescan_remove();
228 }
229
230 mutex_lock(&zbus_list_lock);
231 list_del(&zbus->bus_next);
232 mutex_unlock(&zbus_list_lock);
233 kfree(zbus);
234}
235
236static void zpci_bus_put(struct zpci_bus *zbus)
237{
238 kref_put(&zbus->kref, zpci_bus_release);
239}
240
241static struct zpci_bus *zpci_bus_get(int topo, bool topo_is_tid)
242{
243 struct zpci_bus *zbus;
244
245 mutex_lock(&zbus_list_lock);
246 list_for_each_entry(zbus, &zbus_list, bus_next) {
247 if (!zbus->multifunction)
248 continue;
249 if (topo_is_tid == zbus->topo_is_tid && topo == zbus->topo) {
250 kref_get(&zbus->kref);
251 goto out_unlock;
252 }
253 }
254 zbus = NULL;
255out_unlock:
256 mutex_unlock(&zbus_list_lock);
257 return zbus;
258}
259
260static struct zpci_bus *zpci_bus_alloc(int topo, bool topo_is_tid)
261{
262 struct zpci_bus *zbus;
263
264 zbus = kzalloc(sizeof(*zbus), GFP_KERNEL);
265 if (!zbus)
266 return NULL;
267
268 zbus->topo = topo;
269 zbus->topo_is_tid = topo_is_tid;
270 INIT_LIST_HEAD(&zbus->bus_next);
271 mutex_lock(&zbus_list_lock);
272 list_add_tail(&zbus->bus_next, &zbus_list);
273 mutex_unlock(&zbus_list_lock);
274
275 kref_init(&zbus->kref);
276 INIT_LIST_HEAD(&zbus->resources);
277
278 zbus->bus_resource.start = 0;
279 zbus->bus_resource.end = ZPCI_BUS_NR;
280 zbus->bus_resource.flags = IORESOURCE_BUS;
281 pci_add_resource(&zbus->resources, &zbus->bus_resource);
282
283 return zbus;
284}
285
286void pcibios_bus_add_device(struct pci_dev *pdev)
287{
288 struct zpci_dev *zdev = to_zpci(pdev);
289
290 /*
291 * With pdev->no_vf_scan the common PCI probing code does not
292 * perform PF/VF linking.
293 */
294 if (zdev->vfn) {
295 zpci_iov_setup_virtfn(zdev->zbus, pdev, zdev->vfn);
296 pdev->no_command_memory = 1;
297 }
298}
299
300static int zpci_bus_add_device(struct zpci_bus *zbus, struct zpci_dev *zdev)
301{
302 int rc = -EINVAL;
303
304 if (zbus->multifunction) {
305 if (!zdev->rid_available) {
306 WARN_ONCE(1, "rid_available not set for multifunction\n");
307 return rc;
308 }
309 zdev->devfn = zdev->rid & ZPCI_RID_MASK_DEVFN;
310 }
311
312 if (zbus->function[zdev->devfn]) {
313 pr_err("devfn %04x is already assigned\n", zdev->devfn);
314 return rc;
315 }
316 zdev->zbus = zbus;
317 zbus->function[zdev->devfn] = zdev;
318 zpci_nb_devices++;
319
320 rc = zpci_init_slot(zdev);
321 if (rc)
322 goto error;
323 zdev->has_hp_slot = 1;
324
325 return 0;
326
327error:
328 zbus->function[zdev->devfn] = NULL;
329 zdev->zbus = NULL;
330 zpci_nb_devices--;
331 return rc;
332}
333
334static bool zpci_bus_is_isolated_vf(struct zpci_bus *zbus, struct zpci_dev *zdev)
335{
336 struct pci_dev *pdev;
337
338 pdev = zpci_iov_find_parent_pf(zbus, zdev);
339 if (!pdev)
340 return true;
341 pci_dev_put(pdev);
342 return false;
343}
344
345int zpci_bus_device_register(struct zpci_dev *zdev, struct pci_ops *ops)
346{
347 bool topo_is_tid = zdev->tid_avail;
348 struct zpci_bus *zbus = NULL;
349 int topo, rc = -EBADF;
350
351 if (zpci_nb_devices == ZPCI_NR_DEVICES) {
352 pr_warn("Adding PCI function %08x failed because the configured limit of %d is reached\n",
353 zdev->fid, ZPCI_NR_DEVICES);
354 return -ENOSPC;
355 }
356
357 topo = topo_is_tid ? zdev->tid : zdev->pchid;
358 zbus = zpci_bus_get(topo, topo_is_tid);
359 /*
360 * An isolated VF gets its own domain/bus even if there exists
361 * a matching domain/bus already
362 */
363 if (zbus && zpci_bus_is_isolated_vf(zbus, zdev)) {
364 zpci_bus_put(zbus);
365 zbus = NULL;
366 }
367
368 if (!zbus) {
369 zbus = zpci_bus_alloc(topo, topo_is_tid);
370 if (!zbus)
371 return -ENOMEM;
372 }
373
374 if (!zbus->bus) {
375 /* The UID of the first PCI function registered with a zpci_bus
376 * is used as the domain number for that bus. Currently there
377 * is exactly one zpci_bus per domain.
378 */
379 rc = zpci_bus_create_pci_bus(zbus, zdev, ops);
380 if (rc)
381 goto error;
382 }
383
384 rc = zpci_bus_add_device(zbus, zdev);
385 if (rc)
386 goto error;
387
388 return 0;
389
390error:
391 pr_err("Adding PCI function %08x failed\n", zdev->fid);
392 zpci_bus_put(zbus);
393 return rc;
394}
395
396void zpci_bus_device_unregister(struct zpci_dev *zdev)
397{
398 struct zpci_bus *zbus = zdev->zbus;
399
400 zpci_nb_devices--;
401 zbus->function[zdev->devfn] = NULL;
402 zpci_bus_put(zbus);
403}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright IBM Corp. 2020
4 *
5 * Author(s):
6 * Pierre Morel <pmorel@linux.ibm.com>
7 *
8 */
9
10#define KMSG_COMPONENT "zpci"
11#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
12
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/err.h>
16#include <linux/export.h>
17#include <linux/delay.h>
18#include <linux/seq_file.h>
19#include <linux/jump_label.h>
20#include <linux/pci.h>
21#include <linux/printk.h>
22
23#include <asm/pci_clp.h>
24#include <asm/pci_dma.h>
25
26#include "pci_bus.h"
27
28static LIST_HEAD(zbus_list);
29static DEFINE_SPINLOCK(zbus_list_lock);
30static int zpci_nb_devices;
31
32/* zpci_bus_scan
33 * @zbus: the zbus holding the zdevices
34 * @ops: the pci operations
35 *
36 * The domain number must be set before pci_scan_root_bus is called.
37 * This function can be called once the domain is known, hence
38 * when the function_0 is dicovered.
39 */
40static int zpci_bus_scan(struct zpci_bus *zbus, int domain, struct pci_ops *ops)
41{
42 struct pci_bus *bus;
43 int rc;
44
45 rc = zpci_alloc_domain(domain);
46 if (rc < 0)
47 return rc;
48 zbus->domain_nr = rc;
49
50 bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, ops, zbus, &zbus->resources);
51 if (!bus) {
52 zpci_free_domain(zbus->domain_nr);
53 return -EFAULT;
54 }
55
56 zbus->bus = bus;
57 pci_bus_add_devices(bus);
58 return 0;
59}
60
61static void zpci_bus_release(struct kref *kref)
62{
63 struct zpci_bus *zbus = container_of(kref, struct zpci_bus, kref);
64
65 if (zbus->bus) {
66 pci_lock_rescan_remove();
67 pci_stop_root_bus(zbus->bus);
68
69 zpci_free_domain(zbus->domain_nr);
70 pci_free_resource_list(&zbus->resources);
71
72 pci_remove_root_bus(zbus->bus);
73 pci_unlock_rescan_remove();
74 }
75
76 spin_lock(&zbus_list_lock);
77 list_del(&zbus->bus_next);
78 spin_unlock(&zbus_list_lock);
79 kfree(zbus);
80}
81
82static void zpci_bus_put(struct zpci_bus *zbus)
83{
84 kref_put(&zbus->kref, zpci_bus_release);
85}
86
87static struct zpci_bus *zpci_bus_get(int pchid)
88{
89 struct zpci_bus *zbus;
90
91 spin_lock(&zbus_list_lock);
92 list_for_each_entry(zbus, &zbus_list, bus_next) {
93 if (pchid == zbus->pchid) {
94 kref_get(&zbus->kref);
95 goto out_unlock;
96 }
97 }
98 zbus = NULL;
99out_unlock:
100 spin_unlock(&zbus_list_lock);
101 return zbus;
102}
103
104static struct zpci_bus *zpci_bus_alloc(int pchid)
105{
106 struct zpci_bus *zbus;
107
108 zbus = kzalloc(sizeof(*zbus), GFP_KERNEL);
109 if (!zbus)
110 return NULL;
111
112 zbus->pchid = pchid;
113 INIT_LIST_HEAD(&zbus->bus_next);
114 spin_lock(&zbus_list_lock);
115 list_add_tail(&zbus->bus_next, &zbus_list);
116 spin_unlock(&zbus_list_lock);
117
118 kref_init(&zbus->kref);
119 INIT_LIST_HEAD(&zbus->resources);
120
121 zbus->bus_resource.start = 0;
122 zbus->bus_resource.end = ZPCI_BUS_NR;
123 zbus->bus_resource.flags = IORESOURCE_BUS;
124 pci_add_resource(&zbus->resources, &zbus->bus_resource);
125
126 return zbus;
127}
128
129#ifdef CONFIG_PCI_IOV
130static int zpci_bus_link_virtfn(struct pci_dev *pdev,
131 struct pci_dev *virtfn, int vfid)
132{
133 int rc;
134
135 rc = pci_iov_sysfs_link(pdev, virtfn, vfid);
136 if (rc)
137 return rc;
138
139 virtfn->is_virtfn = 1;
140 virtfn->multifunction = 0;
141 virtfn->physfn = pci_dev_get(pdev);
142
143 return 0;
144}
145
146static int zpci_bus_setup_virtfn(struct zpci_bus *zbus,
147 struct pci_dev *virtfn, int vfn)
148{
149 int i, cand_devfn;
150 struct zpci_dev *zdev;
151 struct pci_dev *pdev;
152 int vfid = vfn - 1; /* Linux' vfid's start at 0 vfn at 1*/
153 int rc = 0;
154
155 if (!zbus->multifunction)
156 return 0;
157
158 /* If the parent PF for the given VF is also configured in the
159 * instance, it must be on the same zbus.
160 * We can then identify the parent PF by checking what
161 * devfn the VF would have if it belonged to that PF using the PF's
162 * stride and offset. Only if this candidate devfn matches the
163 * actual devfn will we link both functions.
164 */
165 for (i = 0; i < ZPCI_FUNCTIONS_PER_BUS; i++) {
166 zdev = zbus->function[i];
167 if (zdev && zdev->is_physfn) {
168 pdev = pci_get_slot(zbus->bus, zdev->devfn);
169 if (!pdev)
170 continue;
171 cand_devfn = pci_iov_virtfn_devfn(pdev, vfid);
172 if (cand_devfn == virtfn->devfn) {
173 rc = zpci_bus_link_virtfn(pdev, virtfn, vfid);
174 /* balance pci_get_slot() */
175 pci_dev_put(pdev);
176 break;
177 }
178 /* balance pci_get_slot() */
179 pci_dev_put(pdev);
180 }
181 }
182 return rc;
183}
184#else
185static inline int zpci_bus_setup_virtfn(struct zpci_bus *zbus,
186 struct pci_dev *virtfn, int vfn)
187{
188 return 0;
189}
190#endif
191
192void pcibios_bus_add_device(struct pci_dev *pdev)
193{
194 struct zpci_dev *zdev = to_zpci(pdev);
195
196 /*
197 * With pdev->no_vf_scan the common PCI probing code does not
198 * perform PF/VF linking.
199 */
200 if (zdev->vfn)
201 zpci_bus_setup_virtfn(zdev->zbus, pdev, zdev->vfn);
202
203}
204
205static int zpci_bus_add_device(struct zpci_bus *zbus, struct zpci_dev *zdev)
206{
207 struct pci_bus *bus;
208 struct resource_entry *window, *n;
209 struct resource *res;
210 struct pci_dev *pdev;
211 int rc;
212
213 bus = zbus->bus;
214 if (!bus)
215 return -EINVAL;
216
217 pdev = pci_get_slot(bus, zdev->devfn);
218 if (pdev) {
219 /* Device is already known. */
220 pci_dev_put(pdev);
221 return 0;
222 }
223
224 rc = zpci_init_slot(zdev);
225 if (rc)
226 return rc;
227 zdev->has_hp_slot = 1;
228
229 resource_list_for_each_entry_safe(window, n, &zbus->resources) {
230 res = window->res;
231 pci_bus_add_resource(bus, res, 0);
232 }
233
234 pdev = pci_scan_single_device(bus, zdev->devfn);
235 if (pdev)
236 pci_bus_add_device(pdev);
237
238 return 0;
239}
240
241static void zpci_bus_add_devices(struct zpci_bus *zbus)
242{
243 int i;
244
245 for (i = 1; i < ZPCI_FUNCTIONS_PER_BUS; i++)
246 if (zbus->function[i])
247 zpci_bus_add_device(zbus, zbus->function[i]);
248
249 pci_lock_rescan_remove();
250 pci_bus_add_devices(zbus->bus);
251 pci_unlock_rescan_remove();
252}
253
254int zpci_bus_device_register(struct zpci_dev *zdev, struct pci_ops *ops)
255{
256 struct zpci_bus *zbus = NULL;
257 int rc = -EBADF;
258
259 if (zpci_nb_devices == ZPCI_NR_DEVICES) {
260 pr_warn("Adding PCI function %08x failed because the configured limit of %d is reached\n",
261 zdev->fid, ZPCI_NR_DEVICES);
262 return -ENOSPC;
263 }
264 zpci_nb_devices++;
265
266 if (zdev->devfn >= ZPCI_FUNCTIONS_PER_BUS)
267 return -EINVAL;
268
269 if (!s390_pci_no_rid && zdev->rid_available)
270 zbus = zpci_bus_get(zdev->pchid);
271
272 if (!zbus) {
273 zbus = zpci_bus_alloc(zdev->pchid);
274 if (!zbus)
275 return -ENOMEM;
276 }
277
278 zdev->zbus = zbus;
279 if (zbus->function[zdev->devfn]) {
280 pr_err("devfn %04x is already assigned\n", zdev->devfn);
281 goto error; /* rc already set */
282 }
283 zbus->function[zdev->devfn] = zdev;
284
285 zpci_setup_bus_resources(zdev, &zbus->resources);
286
287 if (zbus->bus) {
288 if (!zbus->multifunction) {
289 WARN_ONCE(1, "zbus is not multifunction\n");
290 goto error_bus;
291 }
292 if (!zdev->rid_available) {
293 WARN_ONCE(1, "rid_available not set for multifunction\n");
294 goto error_bus;
295 }
296 rc = zpci_bus_add_device(zbus, zdev);
297 if (rc)
298 goto error_bus;
299 } else if (zdev->devfn == 0) {
300 if (zbus->multifunction && !zdev->rid_available) {
301 WARN_ONCE(1, "rid_available not set on function 0 for multifunction\n");
302 goto error_bus;
303 }
304 rc = zpci_bus_scan(zbus, (u16)zdev->uid, ops);
305 if (rc)
306 goto error_bus;
307 zpci_bus_add_devices(zbus);
308 rc = zpci_init_slot(zdev);
309 if (rc)
310 goto error_bus;
311 zdev->has_hp_slot = 1;
312 zbus->multifunction = zdev->rid_available;
313 zbus->max_bus_speed = zdev->max_bus_speed;
314 } else {
315 zbus->multifunction = 1;
316 }
317
318 return 0;
319
320error_bus:
321 zpci_nb_devices--;
322 zbus->function[zdev->devfn] = NULL;
323error:
324 pr_err("Adding PCI function %08x failed\n", zdev->fid);
325 zpci_bus_put(zbus);
326 return rc;
327}
328
329void zpci_bus_device_unregister(struct zpci_dev *zdev)
330{
331 struct zpci_bus *zbus = zdev->zbus;
332
333 zpci_nb_devices--;
334 zbus->function[zdev->devfn] = NULL;
335 zpci_bus_put(zbus);
336}