Loading...
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/* Copyright (c) 2021, Microsoft Corporation. */
3
4#include <linux/delay.h>
5#include <linux/device.h>
6#include <linux/io.h>
7#include <linux/mm.h>
8
9#include <net/mana/shm_channel.h>
10
11#define PAGE_FRAME_L48_WIDTH_BYTES 6
12#define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
13#define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
14#define PAGE_FRAME_H4_WIDTH_BITS 4
15#define VECTOR_MASK 0xFFFF
16#define SHMEM_VF_RESET_STATE ((u32)-1)
17
18#define SMC_MSG_TYPE_ESTABLISH_HWC 1
19#define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
20
21#define SMC_MSG_TYPE_DESTROY_HWC 2
22#define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
23
24#define SMC_MSG_DIRECTION_REQUEST 0
25#define SMC_MSG_DIRECTION_RESPONSE 1
26
27/* Structures labeled with "HW DATA" are exchanged with the hardware. All of
28 * them are naturally aligned and hence don't need __packed.
29 */
30
31/* Shared memory channel protocol header
32 *
33 * msg_type: set on request and response; response matches request.
34 * msg_version: newer PF writes back older response (matching request)
35 * older PF acts on latest version known and sets that version in result
36 * (less than request).
37 * direction: 0 for request, VF->PF; 1 for response, PF->VF.
38 * status: 0 on request,
39 * operation result on response (success = 0, failure = 1 or greater).
40 * reset_vf: If set on either establish or destroy request, indicates perform
41 * FLR before/after the operation.
42 * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
43 */
44union smc_proto_hdr {
45 u32 as_uint32;
46
47 struct {
48 u8 msg_type : 3;
49 u8 msg_version : 3;
50 u8 reserved_1 : 1;
51 u8 direction : 1;
52
53 u8 status;
54
55 u8 reserved_2;
56
57 u8 reset_vf : 1;
58 u8 reserved_3 : 6;
59 u8 owner_is_pf : 1;
60 };
61}; /* HW DATA */
62
63#define SMC_APERTURE_BITS 256
64#define SMC_BASIC_UNIT (sizeof(u32))
65#define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
66#define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
67
68static int mana_smc_poll_register(void __iomem *base, bool reset)
69{
70 void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
71 u32 last_dword;
72 int i;
73
74 /* Poll the hardware for the ownership bit. This should be pretty fast,
75 * but let's do it in a loop just in case the hardware or the PF
76 * driver are temporarily busy.
77 */
78 for (i = 0; i < 20 * 1000; i++) {
79 last_dword = readl(ptr);
80
81 /* shmem reads as 0xFFFFFFFF in the reset case */
82 if (reset && last_dword == SHMEM_VF_RESET_STATE)
83 return 0;
84
85 /* If bit_31 is set, the PF currently owns the SMC. */
86 if (!(last_dword & BIT(31)))
87 return 0;
88
89 usleep_range(1000, 2000);
90 }
91
92 return -ETIMEDOUT;
93}
94
95static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type,
96 u32 msg_version, bool reset_vf)
97{
98 void __iomem *base = sc->base;
99 union smc_proto_hdr hdr;
100 int err;
101
102 /* Wait for PF to respond. */
103 err = mana_smc_poll_register(base, reset_vf);
104 if (err)
105 return err;
106
107 hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
108
109 if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
110 return 0;
111
112 /* Validate protocol fields from the PF driver */
113 if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
114 hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
115 dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n",
116 hdr.as_uint32, msg_type, msg_version);
117 return -EPROTO;
118 }
119
120 /* Validate the operation result */
121 if (hdr.status != 0) {
122 dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status);
123 return -EPROTO;
124 }
125
126 return 0;
127}
128
129void mana_smc_init(struct shm_channel *sc, struct device *dev,
130 void __iomem *base)
131{
132 sc->dev = dev;
133 sc->base = base;
134}
135
136int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr,
137 u64 cq_addr, u64 rq_addr, u64 sq_addr,
138 u32 eq_msix_index)
139{
140 union smc_proto_hdr *hdr;
141 u16 all_addr_h4bits = 0;
142 u16 frame_addr_seq = 0;
143 u64 frame_addr = 0;
144 u8 shm_buf[32];
145 u64 *shmem;
146 u32 *dword;
147 u8 *ptr;
148 int err;
149 int i;
150
151 /* Ensure VF already has possession of shared memory */
152 err = mana_smc_poll_register(sc->base, false);
153 if (err) {
154 dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err);
155 return err;
156 }
157
158 if (!PAGE_ALIGNED(eq_addr) || !PAGE_ALIGNED(cq_addr) ||
159 !PAGE_ALIGNED(rq_addr) || !PAGE_ALIGNED(sq_addr))
160 return -EINVAL;
161
162 if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
163 return -EINVAL;
164
165 /* Scheme for packing four addresses and extra info into 256 bits.
166 *
167 * Addresses must be page frame aligned, so only frame address bits
168 * are transferred.
169 *
170 * 52-bit frame addresses are split into the lower 48 bits and upper
171 * 4 bits. Lower 48 bits of 4 address are written sequentially from
172 * the start of the 256-bit shared memory region followed by 16 bits
173 * containing the upper 4 bits of the 4 addresses in sequence.
174 *
175 * A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
176 *
177 * The final 32-bit dword is used for protocol control information as
178 * defined in smc_proto_hdr.
179 */
180
181 memset(shm_buf, 0, sizeof(shm_buf));
182 ptr = shm_buf;
183
184 /* EQ addr: low 48 bits of frame address */
185 shmem = (u64 *)ptr;
186 frame_addr = PHYS_PFN(eq_addr);
187 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
188 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
189 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
190 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
191
192 /* CQ addr: low 48 bits of frame address */
193 shmem = (u64 *)ptr;
194 frame_addr = PHYS_PFN(cq_addr);
195 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
196 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
197 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
198 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
199
200 /* RQ addr: low 48 bits of frame address */
201 shmem = (u64 *)ptr;
202 frame_addr = PHYS_PFN(rq_addr);
203 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
204 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
205 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
206 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
207
208 /* SQ addr: low 48 bits of frame address */
209 shmem = (u64 *)ptr;
210 frame_addr = PHYS_PFN(sq_addr);
211 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
212 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
213 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
214 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
215
216 /* High 4 bits of the four frame addresses */
217 *((u16 *)ptr) = all_addr_h4bits;
218 ptr += sizeof(u16);
219
220 /* EQ MSIX vector number */
221 *((u16 *)ptr) = (u16)eq_msix_index;
222 ptr += sizeof(u16);
223
224 /* 32-bit protocol header in final dword */
225 *((u32 *)ptr) = 0;
226
227 hdr = (union smc_proto_hdr *)ptr;
228 hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
229 hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
230 hdr->direction = SMC_MSG_DIRECTION_REQUEST;
231 hdr->reset_vf = reset_vf;
232
233 /* Write 256-message buffer to shared memory (final 32-bit write
234 * triggers HW to set possession bit to PF).
235 */
236 dword = (u32 *)shm_buf;
237 for (i = 0; i < SMC_APERTURE_DWORDS; i++)
238 writel(*dword++, sc->base + i * SMC_BASIC_UNIT);
239
240 /* Read shmem response (polling for VF possession) and validate.
241 * For setup, waiting for response on shared memory is not strictly
242 * necessary, since wait occurs later for results to appear in EQE's.
243 */
244 err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
245 SMC_MSG_TYPE_ESTABLISH_HWC_VERSION,
246 reset_vf);
247 if (err) {
248 dev_err(sc->dev, "Error when setting up HWC: %d\n", err);
249 return err;
250 }
251
252 return 0;
253}
254
255int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
256{
257 union smc_proto_hdr hdr = {};
258 int err;
259
260 /* Ensure already has possession of shared memory */
261 err = mana_smc_poll_register(sc->base, false);
262 if (err) {
263 dev_err(sc->dev, "Timeout when tearing down HWC\n");
264 return err;
265 }
266
267 /* Set up protocol header for HWC destroy message */
268 hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
269 hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
270 hdr.direction = SMC_MSG_DIRECTION_REQUEST;
271 hdr.reset_vf = reset_vf;
272
273 /* Write message in high 32 bits of 256-bit shared memory, causing HW
274 * to set possession bit to PF.
275 */
276 writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
277
278 /* Read shmem response (polling for VF possession) and validate.
279 * For teardown, waiting for response is required to ensure hardware
280 * invalidates MST entries before software frees memory.
281 */
282 err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
283 SMC_MSG_TYPE_DESTROY_HWC_VERSION,
284 reset_vf);
285 if (err) {
286 dev_err(sc->dev, "Error when tearing down HWC: %d\n", err);
287 return err;
288 }
289
290 return 0;
291}
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/* Copyright (c) 2021, Microsoft Corporation. */
3
4#include <linux/delay.h>
5#include <linux/device.h>
6#include <linux/io.h>
7#include <linux/mm.h>
8
9#include <net/mana/gdma.h>
10#include <net/mana/shm_channel.h>
11
12#define PAGE_FRAME_L48_WIDTH_BYTES 6
13#define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
14#define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
15#define PAGE_FRAME_H4_WIDTH_BITS 4
16#define VECTOR_MASK 0xFFFF
17#define SHMEM_VF_RESET_STATE ((u32)-1)
18
19#define SMC_MSG_TYPE_ESTABLISH_HWC 1
20#define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
21
22#define SMC_MSG_TYPE_DESTROY_HWC 2
23#define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
24
25#define SMC_MSG_DIRECTION_REQUEST 0
26#define SMC_MSG_DIRECTION_RESPONSE 1
27
28/* Structures labeled with "HW DATA" are exchanged with the hardware. All of
29 * them are naturally aligned and hence don't need __packed.
30 */
31
32/* Shared memory channel protocol header
33 *
34 * msg_type: set on request and response; response matches request.
35 * msg_version: newer PF writes back older response (matching request)
36 * older PF acts on latest version known and sets that version in result
37 * (less than request).
38 * direction: 0 for request, VF->PF; 1 for response, PF->VF.
39 * status: 0 on request,
40 * operation result on response (success = 0, failure = 1 or greater).
41 * reset_vf: If set on either establish or destroy request, indicates perform
42 * FLR before/after the operation.
43 * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
44 */
45union smc_proto_hdr {
46 u32 as_uint32;
47
48 struct {
49 u8 msg_type : 3;
50 u8 msg_version : 3;
51 u8 reserved_1 : 1;
52 u8 direction : 1;
53
54 u8 status;
55
56 u8 reserved_2;
57
58 u8 reset_vf : 1;
59 u8 reserved_3 : 6;
60 u8 owner_is_pf : 1;
61 };
62}; /* HW DATA */
63
64#define SMC_APERTURE_BITS 256
65#define SMC_BASIC_UNIT (sizeof(u32))
66#define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
67#define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
68
69static int mana_smc_poll_register(void __iomem *base, bool reset)
70{
71 void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
72 u32 last_dword;
73 int i;
74
75 /* Poll the hardware for the ownership bit. This should be pretty fast,
76 * but let's do it in a loop just in case the hardware or the PF
77 * driver are temporarily busy.
78 */
79 for (i = 0; i < 20 * 1000; i++) {
80 last_dword = readl(ptr);
81
82 /* shmem reads as 0xFFFFFFFF in the reset case */
83 if (reset && last_dword == SHMEM_VF_RESET_STATE)
84 return 0;
85
86 /* If bit_31 is set, the PF currently owns the SMC. */
87 if (!(last_dword & BIT(31)))
88 return 0;
89
90 usleep_range(1000, 2000);
91 }
92
93 return -ETIMEDOUT;
94}
95
96static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type,
97 u32 msg_version, bool reset_vf)
98{
99 void __iomem *base = sc->base;
100 union smc_proto_hdr hdr;
101 int err;
102
103 /* Wait for PF to respond. */
104 err = mana_smc_poll_register(base, reset_vf);
105 if (err)
106 return err;
107
108 hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
109
110 if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
111 return 0;
112
113 /* Validate protocol fields from the PF driver */
114 if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
115 hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
116 dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n",
117 hdr.as_uint32, msg_type, msg_version);
118 return -EPROTO;
119 }
120
121 /* Validate the operation result */
122 if (hdr.status != 0) {
123 dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status);
124 return -EPROTO;
125 }
126
127 return 0;
128}
129
130void mana_smc_init(struct shm_channel *sc, struct device *dev,
131 void __iomem *base)
132{
133 sc->dev = dev;
134 sc->base = base;
135}
136
137int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr,
138 u64 cq_addr, u64 rq_addr, u64 sq_addr,
139 u32 eq_msix_index)
140{
141 union smc_proto_hdr *hdr;
142 u16 all_addr_h4bits = 0;
143 u16 frame_addr_seq = 0;
144 u64 frame_addr = 0;
145 u8 shm_buf[32];
146 u64 *shmem;
147 u32 *dword;
148 u8 *ptr;
149 int err;
150 int i;
151
152 /* Ensure VF already has possession of shared memory */
153 err = mana_smc_poll_register(sc->base, false);
154 if (err) {
155 dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err);
156 return err;
157 }
158
159 if (!MANA_PAGE_ALIGNED(eq_addr) || !MANA_PAGE_ALIGNED(cq_addr) ||
160 !MANA_PAGE_ALIGNED(rq_addr) || !MANA_PAGE_ALIGNED(sq_addr))
161 return -EINVAL;
162
163 if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
164 return -EINVAL;
165
166 /* Scheme for packing four addresses and extra info into 256 bits.
167 *
168 * Addresses must be page frame aligned, so only frame address bits
169 * are transferred.
170 *
171 * 52-bit frame addresses are split into the lower 48 bits and upper
172 * 4 bits. Lower 48 bits of 4 address are written sequentially from
173 * the start of the 256-bit shared memory region followed by 16 bits
174 * containing the upper 4 bits of the 4 addresses in sequence.
175 *
176 * A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
177 *
178 * The final 32-bit dword is used for protocol control information as
179 * defined in smc_proto_hdr.
180 */
181
182 memset(shm_buf, 0, sizeof(shm_buf));
183 ptr = shm_buf;
184
185 /* EQ addr: low 48 bits of frame address */
186 shmem = (u64 *)ptr;
187 frame_addr = MANA_PFN(eq_addr);
188 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
189 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
190 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
191 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
192
193 /* CQ addr: low 48 bits of frame address */
194 shmem = (u64 *)ptr;
195 frame_addr = MANA_PFN(cq_addr);
196 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
197 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
198 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
199 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
200
201 /* RQ addr: low 48 bits of frame address */
202 shmem = (u64 *)ptr;
203 frame_addr = MANA_PFN(rq_addr);
204 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
205 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
206 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
207 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
208
209 /* SQ addr: low 48 bits of frame address */
210 shmem = (u64 *)ptr;
211 frame_addr = MANA_PFN(sq_addr);
212 *shmem = frame_addr & PAGE_FRAME_L48_MASK;
213 all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
214 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
215 ptr += PAGE_FRAME_L48_WIDTH_BYTES;
216
217 /* High 4 bits of the four frame addresses */
218 *((u16 *)ptr) = all_addr_h4bits;
219 ptr += sizeof(u16);
220
221 /* EQ MSIX vector number */
222 *((u16 *)ptr) = (u16)eq_msix_index;
223 ptr += sizeof(u16);
224
225 /* 32-bit protocol header in final dword */
226 *((u32 *)ptr) = 0;
227
228 hdr = (union smc_proto_hdr *)ptr;
229 hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
230 hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
231 hdr->direction = SMC_MSG_DIRECTION_REQUEST;
232 hdr->reset_vf = reset_vf;
233
234 /* Write 256-message buffer to shared memory (final 32-bit write
235 * triggers HW to set possession bit to PF).
236 */
237 dword = (u32 *)shm_buf;
238 for (i = 0; i < SMC_APERTURE_DWORDS; i++)
239 writel(*dword++, sc->base + i * SMC_BASIC_UNIT);
240
241 /* Read shmem response (polling for VF possession) and validate.
242 * For setup, waiting for response on shared memory is not strictly
243 * necessary, since wait occurs later for results to appear in EQE's.
244 */
245 err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
246 SMC_MSG_TYPE_ESTABLISH_HWC_VERSION,
247 reset_vf);
248 if (err) {
249 dev_err(sc->dev, "Error when setting up HWC: %d\n", err);
250 return err;
251 }
252
253 return 0;
254}
255
256int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
257{
258 union smc_proto_hdr hdr = {};
259 int err;
260
261 /* Ensure already has possession of shared memory */
262 err = mana_smc_poll_register(sc->base, false);
263 if (err) {
264 dev_err(sc->dev, "Timeout when tearing down HWC\n");
265 return err;
266 }
267
268 /* Set up protocol header for HWC destroy message */
269 hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
270 hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
271 hdr.direction = SMC_MSG_DIRECTION_REQUEST;
272 hdr.reset_vf = reset_vf;
273
274 /* Write message in high 32 bits of 256-bit shared memory, causing HW
275 * to set possession bit to PF.
276 */
277 writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
278
279 /* Read shmem response (polling for VF possession) and validate.
280 * For teardown, waiting for response is required to ensure hardware
281 * invalidates MST entries before software frees memory.
282 */
283 err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
284 SMC_MSG_TYPE_DESTROY_HWC_VERSION,
285 reset_vf);
286 if (err) {
287 dev_err(sc->dev, "Error when tearing down HWC: %d\n", err);
288 return err;
289 }
290
291 return 0;
292}