Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2016-2019 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8#include "habanalabs.h"
9#include "../include/hw_ip/pci/pci_general.h"
10
11#include <linux/pci.h>
12#include <linux/bitfield.h>
13
14#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 10)
15
16#define IATU_REGION_CTRL_REGION_EN_MASK BIT(31)
17#define IATU_REGION_CTRL_MATCH_MODE_MASK BIT(30)
18#define IATU_REGION_CTRL_NUM_MATCH_EN_MASK BIT(19)
19#define IATU_REGION_CTRL_BAR_NUM_MASK GENMASK(10, 8)
20
21/**
22 * hl_pci_bars_map() - Map PCI BARs.
23 * @hdev: Pointer to hl_device structure.
24 * @name: Array of BAR names.
25 * @is_wc: Array with flag per BAR whether a write-combined mapping is needed.
26 *
27 * Request PCI regions and map them to kernel virtual addresses.
28 *
29 * Return: 0 on success, non-zero for failure.
30 */
31int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
32 bool is_wc[3])
33{
34 struct pci_dev *pdev = hdev->pdev;
35 int rc, i, bar;
36
37 rc = pci_request_regions(pdev, HL_NAME);
38 if (rc) {
39 dev_err(hdev->dev, "Cannot obtain PCI resources\n");
40 return rc;
41 }
42
43 for (i = 0 ; i < 3 ; i++) {
44 bar = i * 2; /* 64-bit BARs */
45 hdev->pcie_bar[bar] = is_wc[i] ?
46 pci_ioremap_wc_bar(pdev, bar) :
47 pci_ioremap_bar(pdev, bar);
48 if (!hdev->pcie_bar[bar]) {
49 dev_err(hdev->dev, "pci_ioremap%s_bar failed for %s\n",
50 is_wc[i] ? "_wc" : "", name[i]);
51 rc = -ENODEV;
52 goto err;
53 }
54 }
55
56 return 0;
57
58err:
59 for (i = 2 ; i >= 0 ; i--) {
60 bar = i * 2; /* 64-bit BARs */
61 if (hdev->pcie_bar[bar])
62 iounmap(hdev->pcie_bar[bar]);
63 }
64
65 pci_release_regions(pdev);
66
67 return rc;
68}
69
70/**
71 * hl_pci_bars_unmap() - Unmap PCI BARS.
72 * @hdev: Pointer to hl_device structure.
73 *
74 * Release all PCI BARs and unmap their virtual addresses.
75 */
76static void hl_pci_bars_unmap(struct hl_device *hdev)
77{
78 struct pci_dev *pdev = hdev->pdev;
79 int i, bar;
80
81 for (i = 2 ; i >= 0 ; i--) {
82 bar = i * 2; /* 64-bit BARs */
83 iounmap(hdev->pcie_bar[bar]);
84 }
85
86 pci_release_regions(pdev);
87}
88
89/**
90 * hl_pci_elbi_write() - Write through the ELBI interface.
91 * @hdev: Pointer to hl_device structure.
92 * @addr: Address to write to
93 * @data: Data to write
94 *
95 * Return: 0 on success, negative value for failure.
96 */
97static int hl_pci_elbi_write(struct hl_device *hdev, u64 addr, u32 data)
98{
99 struct pci_dev *pdev = hdev->pdev;
100 ktime_t timeout;
101 u64 msec;
102 u32 val;
103
104 if (hdev->pldm)
105 msec = HL_PLDM_PCI_ELBI_TIMEOUT_MSEC;
106 else
107 msec = HL_PCI_ELBI_TIMEOUT_MSEC;
108
109 /* Clear previous status */
110 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, 0);
111
112 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_ADDR, (u32) addr);
113 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_DATA, data);
114 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_CTRL,
115 PCI_CONFIG_ELBI_CTRL_WRITE);
116
117 timeout = ktime_add_ms(ktime_get(), msec);
118 for (;;) {
119 pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, &val);
120 if (val & PCI_CONFIG_ELBI_STS_MASK)
121 break;
122 if (ktime_compare(ktime_get(), timeout) > 0) {
123 pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS,
124 &val);
125 break;
126 }
127
128 usleep_range(300, 500);
129 }
130
131 if ((val & PCI_CONFIG_ELBI_STS_MASK) == PCI_CONFIG_ELBI_STS_DONE)
132 return 0;
133
134 if (val & PCI_CONFIG_ELBI_STS_ERR) {
135 dev_err(hdev->dev, "Error writing to ELBI\n");
136 return -EIO;
137 }
138
139 if (!(val & PCI_CONFIG_ELBI_STS_MASK)) {
140 dev_err(hdev->dev, "ELBI write didn't finish in time\n");
141 return -EIO;
142 }
143
144 dev_err(hdev->dev, "ELBI write has undefined bits in status\n");
145 return -EIO;
146}
147
148/**
149 * hl_pci_iatu_write() - iatu write routine.
150 * @hdev: Pointer to hl_device structure.
151 * @addr: Address to write to
152 * @data: Data to write
153 *
154 * Return: 0 on success, negative value for failure.
155 */
156int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data)
157{
158 struct asic_fixed_properties *prop = &hdev->asic_prop;
159 u32 dbi_offset;
160 int rc;
161
162 dbi_offset = addr & 0xFFF;
163
164 rc = hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0x00300000);
165 rc |= hl_pci_elbi_write(hdev, prop->pcie_dbi_base_address + dbi_offset,
166 data);
167
168 if (rc)
169 return -EIO;
170
171 return 0;
172}
173
174/**
175 * hl_pci_reset_link_through_bridge() - Reset PCI link.
176 * @hdev: Pointer to hl_device structure.
177 */
178static void hl_pci_reset_link_through_bridge(struct hl_device *hdev)
179{
180 struct pci_dev *pdev = hdev->pdev;
181 struct pci_dev *parent_port;
182 u16 val;
183
184 parent_port = pdev->bus->self;
185 pci_read_config_word(parent_port, PCI_BRIDGE_CONTROL, &val);
186 val |= PCI_BRIDGE_CTL_BUS_RESET;
187 pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
188 ssleep(1);
189
190 val &= ~(PCI_BRIDGE_CTL_BUS_RESET);
191 pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
192 ssleep(3);
193}
194
195/**
196 * hl_pci_set_inbound_region() - Configure inbound region
197 * @hdev: Pointer to hl_device structure.
198 * @region: Inbound region number.
199 * @pci_region: Inbound region parameters.
200 *
201 * Configure the iATU inbound region.
202 *
203 * Return: 0 on success, negative value for failure.
204 */
205int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region,
206 struct hl_inbound_pci_region *pci_region)
207{
208 struct asic_fixed_properties *prop = &hdev->asic_prop;
209 u64 bar_phys_base, region_base, region_end_address;
210 u32 offset, ctrl_reg_val;
211 int rc = 0;
212
213 /* region offset */
214 offset = (0x200 * region) + 0x100;
215
216 if (pci_region->mode == PCI_ADDRESS_MATCH_MODE) {
217 bar_phys_base = hdev->pcie_bar_phys[pci_region->bar];
218 region_base = bar_phys_base + pci_region->offset_in_bar;
219 region_end_address = region_base + pci_region->size - 1;
220
221 rc |= hl_pci_iatu_write(hdev, offset + 0x8,
222 lower_32_bits(region_base));
223 rc |= hl_pci_iatu_write(hdev, offset + 0xC,
224 upper_32_bits(region_base));
225 rc |= hl_pci_iatu_write(hdev, offset + 0x10,
226 lower_32_bits(region_end_address));
227 }
228
229 /* Point to the specified address */
230 rc |= hl_pci_iatu_write(hdev, offset + 0x14,
231 lower_32_bits(pci_region->addr));
232 rc |= hl_pci_iatu_write(hdev, offset + 0x18,
233 upper_32_bits(pci_region->addr));
234 rc |= hl_pci_iatu_write(hdev, offset + 0x0, 0);
235
236 /* Enable + bar/address match + match enable + bar number */
237 ctrl_reg_val = FIELD_PREP(IATU_REGION_CTRL_REGION_EN_MASK, 1);
238 ctrl_reg_val |= FIELD_PREP(IATU_REGION_CTRL_MATCH_MODE_MASK,
239 pci_region->mode);
240 ctrl_reg_val |= FIELD_PREP(IATU_REGION_CTRL_NUM_MATCH_EN_MASK, 1);
241
242 if (pci_region->mode == PCI_BAR_MATCH_MODE)
243 ctrl_reg_val |= FIELD_PREP(IATU_REGION_CTRL_BAR_NUM_MASK,
244 pci_region->bar);
245
246 rc |= hl_pci_iatu_write(hdev, offset + 0x4, ctrl_reg_val);
247
248 /* Return the DBI window to the default location */
249 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
250 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
251
252 if (rc)
253 dev_err(hdev->dev, "failed to map bar %u to 0x%08llx\n",
254 pci_region->bar, pci_region->addr);
255
256 return rc;
257}
258
259/**
260 * hl_pci_set_outbound_region() - Configure outbound region 0
261 * @hdev: Pointer to hl_device structure.
262 * @pci_region: Outbound region parameters.
263 *
264 * Configure the iATU outbound region 0.
265 *
266 * Return: 0 on success, negative value for failure.
267 */
268int hl_pci_set_outbound_region(struct hl_device *hdev,
269 struct hl_outbound_pci_region *pci_region)
270{
271 struct asic_fixed_properties *prop = &hdev->asic_prop;
272 u64 outbound_region_end_address;
273 int rc = 0;
274
275 /* Outbound Region 0 */
276 outbound_region_end_address =
277 pci_region->addr + pci_region->size - 1;
278 rc |= hl_pci_iatu_write(hdev, 0x008,
279 lower_32_bits(pci_region->addr));
280 rc |= hl_pci_iatu_write(hdev, 0x00C,
281 upper_32_bits(pci_region->addr));
282 rc |= hl_pci_iatu_write(hdev, 0x010,
283 lower_32_bits(outbound_region_end_address));
284 rc |= hl_pci_iatu_write(hdev, 0x014, 0);
285
286 if ((hdev->power9_64bit_dma_enable) && (hdev->dma_mask == 64))
287 rc |= hl_pci_iatu_write(hdev, 0x018, 0x08000000);
288 else
289 rc |= hl_pci_iatu_write(hdev, 0x018, 0);
290
291 rc |= hl_pci_iatu_write(hdev, 0x020,
292 upper_32_bits(outbound_region_end_address));
293 /* Increase region size */
294 rc |= hl_pci_iatu_write(hdev, 0x000, 0x00002000);
295 /* Enable */
296 rc |= hl_pci_iatu_write(hdev, 0x004, 0x80000000);
297
298 /* Return the DBI window to the default location */
299 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
300 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
301
302 return rc;
303}
304
305/**
306 * hl_pci_set_dma_mask() - Set DMA masks for the device.
307 * @hdev: Pointer to hl_device structure.
308 *
309 * This function sets the DMA masks (regular and consistent) for a specified
310 * value. If it doesn't succeed, it tries to set it to a fall-back value
311 *
312 * Return: 0 on success, non-zero for failure.
313 */
314static int hl_pci_set_dma_mask(struct hl_device *hdev)
315{
316 struct pci_dev *pdev = hdev->pdev;
317 int rc;
318
319 /* set DMA mask */
320 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask));
321 if (rc) {
322 dev_err(hdev->dev,
323 "Failed to set pci dma mask to %d bits, error %d\n",
324 hdev->dma_mask, rc);
325 return rc;
326 }
327
328 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask));
329 if (rc) {
330 dev_err(hdev->dev,
331 "Failed to set pci consistent dma mask to %d bits, error %d\n",
332 hdev->dma_mask, rc);
333 return rc;
334 }
335
336 return 0;
337}
338
339/**
340 * hl_pci_init() - PCI initialization code.
341 * @hdev: Pointer to hl_device structure.
342 *
343 * Set DMA masks, initialize the PCI controller and map the PCI BARs.
344 *
345 * Return: 0 on success, non-zero for failure.
346 */
347int hl_pci_init(struct hl_device *hdev)
348{
349 struct pci_dev *pdev = hdev->pdev;
350 int rc;
351
352 if (hdev->reset_pcilink)
353 hl_pci_reset_link_through_bridge(hdev);
354
355 rc = pci_enable_device_mem(pdev);
356 if (rc) {
357 dev_err(hdev->dev, "can't enable PCI device\n");
358 return rc;
359 }
360
361 pci_set_master(pdev);
362
363 rc = hdev->asic_funcs->pci_bars_map(hdev);
364 if (rc) {
365 dev_err(hdev->dev, "Failed to initialize PCI BARs\n");
366 goto disable_device;
367 }
368
369 rc = hdev->asic_funcs->init_iatu(hdev);
370 if (rc) {
371 dev_err(hdev->dev, "Failed to initialize iATU\n");
372 goto unmap_pci_bars;
373 }
374
375 rc = hl_pci_set_dma_mask(hdev);
376 if (rc)
377 goto unmap_pci_bars;
378
379 return 0;
380
381unmap_pci_bars:
382 hl_pci_bars_unmap(hdev);
383disable_device:
384 pci_clear_master(pdev);
385 pci_disable_device(pdev);
386
387 return rc;
388}
389
390/**
391 * hl_fw_fini() - PCI finalization code.
392 * @hdev: Pointer to hl_device structure
393 *
394 * Unmap PCI bars and disable PCI device.
395 */
396void hl_pci_fini(struct hl_device *hdev)
397{
398 hl_pci_bars_unmap(hdev);
399
400 pci_clear_master(hdev->pdev);
401 pci_disable_device(hdev->pdev);
402}