Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: (GPL-2.0 OR MIT)
2/* Google virtual Ethernet (gve) driver
3 *
4 * Copyright (C) 2015-2021 Google, Inc.
5 */
6
7#include <linux/etherdevice.h>
8#include <linux/pci.h>
9#include "gve.h"
10#include "gve_adminq.h"
11#include "gve_register.h"
12
13#define GVE_MAX_ADMINQ_RELEASE_CHECK 500
14#define GVE_ADMINQ_SLEEP_LEN 20
15#define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 100
16
17#define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
18"Expected: length=%d, feature_mask=%x.\n" \
19"Actual: length=%d, feature_mask=%x.\n"
20
21#define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n"
22
23static
24struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor,
25 struct gve_device_option *option)
26{
27 void *option_end, *descriptor_end;
28
29 option_end = (void *)(option + 1) + be16_to_cpu(option->option_length);
30 descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length);
31
32 return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end;
33}
34
35#define GVE_DEVICE_OPTION_NO_MIN_RING_SIZE 8
36
37static
38void gve_parse_device_option(struct gve_priv *priv,
39 struct gve_device_descriptor *device_descriptor,
40 struct gve_device_option *option,
41 struct gve_device_option_gqi_rda **dev_op_gqi_rda,
42 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
43 struct gve_device_option_dqo_rda **dev_op_dqo_rda,
44 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
45 struct gve_device_option_dqo_qpl **dev_op_dqo_qpl,
46 struct gve_device_option_buffer_sizes **dev_op_buffer_sizes,
47 struct gve_device_option_flow_steering **dev_op_flow_steering,
48 struct gve_device_option_rss_config **dev_op_rss_config,
49 struct gve_device_option_modify_ring **dev_op_modify_ring)
50{
51 u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
52 u16 option_length = be16_to_cpu(option->option_length);
53 u16 option_id = be16_to_cpu(option->option_id);
54
55 /* If the length or feature mask doesn't match, continue without
56 * enabling the feature.
57 */
58 switch (option_id) {
59 case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING:
60 if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING ||
61 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) {
62 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
63 "Raw Addressing",
64 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING,
65 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING,
66 option_length, req_feat_mask);
67 break;
68 }
69
70 dev_info(&priv->pdev->dev,
71 "Gqi raw addressing device option enabled.\n");
72 priv->queue_format = GVE_GQI_RDA_FORMAT;
73 break;
74 case GVE_DEV_OPT_ID_GQI_RDA:
75 if (option_length < sizeof(**dev_op_gqi_rda) ||
76 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) {
77 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
78 "GQI RDA", (int)sizeof(**dev_op_gqi_rda),
79 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA,
80 option_length, req_feat_mask);
81 break;
82 }
83
84 if (option_length > sizeof(**dev_op_gqi_rda)) {
85 dev_warn(&priv->pdev->dev,
86 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA");
87 }
88 *dev_op_gqi_rda = (void *)(option + 1);
89 break;
90 case GVE_DEV_OPT_ID_GQI_QPL:
91 if (option_length < sizeof(**dev_op_gqi_qpl) ||
92 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
93 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
94 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
95 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
96 option_length, req_feat_mask);
97 break;
98 }
99
100 if (option_length > sizeof(**dev_op_gqi_qpl)) {
101 dev_warn(&priv->pdev->dev,
102 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL");
103 }
104 *dev_op_gqi_qpl = (void *)(option + 1);
105 break;
106 case GVE_DEV_OPT_ID_DQO_RDA:
107 if (option_length < sizeof(**dev_op_dqo_rda) ||
108 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
109 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
110 "DQO RDA", (int)sizeof(**dev_op_dqo_rda),
111 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
112 option_length, req_feat_mask);
113 break;
114 }
115
116 if (option_length > sizeof(**dev_op_dqo_rda)) {
117 dev_warn(&priv->pdev->dev,
118 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA");
119 }
120 *dev_op_dqo_rda = (void *)(option + 1);
121 break;
122 case GVE_DEV_OPT_ID_DQO_QPL:
123 if (option_length < sizeof(**dev_op_dqo_qpl) ||
124 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL) {
125 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
126 "DQO QPL", (int)sizeof(**dev_op_dqo_qpl),
127 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL,
128 option_length, req_feat_mask);
129 break;
130 }
131
132 if (option_length > sizeof(**dev_op_dqo_qpl)) {
133 dev_warn(&priv->pdev->dev,
134 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO QPL");
135 }
136 *dev_op_dqo_qpl = (void *)(option + 1);
137 break;
138 case GVE_DEV_OPT_ID_JUMBO_FRAMES:
139 if (option_length < sizeof(**dev_op_jumbo_frames) ||
140 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
141 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
142 "Jumbo Frames",
143 (int)sizeof(**dev_op_jumbo_frames),
144 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
145 option_length, req_feat_mask);
146 break;
147 }
148
149 if (option_length > sizeof(**dev_op_jumbo_frames)) {
150 dev_warn(&priv->pdev->dev,
151 GVE_DEVICE_OPTION_TOO_BIG_FMT,
152 "Jumbo Frames");
153 }
154 *dev_op_jumbo_frames = (void *)(option + 1);
155 break;
156 case GVE_DEV_OPT_ID_BUFFER_SIZES:
157 if (option_length < sizeof(**dev_op_buffer_sizes) ||
158 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_BUFFER_SIZES) {
159 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
160 "Buffer Sizes",
161 (int)sizeof(**dev_op_buffer_sizes),
162 GVE_DEV_OPT_REQ_FEAT_MASK_BUFFER_SIZES,
163 option_length, req_feat_mask);
164 break;
165 }
166
167 if (option_length > sizeof(**dev_op_buffer_sizes))
168 dev_warn(&priv->pdev->dev,
169 GVE_DEVICE_OPTION_TOO_BIG_FMT,
170 "Buffer Sizes");
171 *dev_op_buffer_sizes = (void *)(option + 1);
172 break;
173 case GVE_DEV_OPT_ID_MODIFY_RING:
174 if (option_length < GVE_DEVICE_OPTION_NO_MIN_RING_SIZE ||
175 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING) {
176 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
177 "Modify Ring", (int)sizeof(**dev_op_modify_ring),
178 GVE_DEV_OPT_REQ_FEAT_MASK_MODIFY_RING,
179 option_length, req_feat_mask);
180 break;
181 }
182
183 if (option_length > sizeof(**dev_op_modify_ring)) {
184 dev_warn(&priv->pdev->dev,
185 GVE_DEVICE_OPTION_TOO_BIG_FMT, "Modify Ring");
186 }
187
188 *dev_op_modify_ring = (void *)(option + 1);
189
190 /* device has not provided min ring size */
191 if (option_length == GVE_DEVICE_OPTION_NO_MIN_RING_SIZE)
192 priv->default_min_ring_size = true;
193 break;
194 case GVE_DEV_OPT_ID_FLOW_STEERING:
195 if (option_length < sizeof(**dev_op_flow_steering) ||
196 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_FLOW_STEERING) {
197 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
198 "Flow Steering",
199 (int)sizeof(**dev_op_flow_steering),
200 GVE_DEV_OPT_REQ_FEAT_MASK_FLOW_STEERING,
201 option_length, req_feat_mask);
202 break;
203 }
204
205 if (option_length > sizeof(**dev_op_flow_steering))
206 dev_warn(&priv->pdev->dev,
207 GVE_DEVICE_OPTION_TOO_BIG_FMT,
208 "Flow Steering");
209 *dev_op_flow_steering = (void *)(option + 1);
210 break;
211 case GVE_DEV_OPT_ID_RSS_CONFIG:
212 if (option_length < sizeof(**dev_op_rss_config) ||
213 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_RSS_CONFIG) {
214 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
215 "RSS config",
216 (int)sizeof(**dev_op_rss_config),
217 GVE_DEV_OPT_REQ_FEAT_MASK_RSS_CONFIG,
218 option_length, req_feat_mask);
219 break;
220 }
221
222 if (option_length > sizeof(**dev_op_rss_config))
223 dev_warn(&priv->pdev->dev,
224 GVE_DEVICE_OPTION_TOO_BIG_FMT,
225 "RSS config");
226 *dev_op_rss_config = (void *)(option + 1);
227 break;
228 default:
229 /* If we don't recognize the option just continue
230 * without doing anything.
231 */
232 dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n",
233 option_id);
234 }
235}
236
237/* Process all device options for a given describe device call. */
238static int
239gve_process_device_options(struct gve_priv *priv,
240 struct gve_device_descriptor *descriptor,
241 struct gve_device_option_gqi_rda **dev_op_gqi_rda,
242 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
243 struct gve_device_option_dqo_rda **dev_op_dqo_rda,
244 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
245 struct gve_device_option_dqo_qpl **dev_op_dqo_qpl,
246 struct gve_device_option_buffer_sizes **dev_op_buffer_sizes,
247 struct gve_device_option_flow_steering **dev_op_flow_steering,
248 struct gve_device_option_rss_config **dev_op_rss_config,
249 struct gve_device_option_modify_ring **dev_op_modify_ring)
250{
251 const int num_options = be16_to_cpu(descriptor->num_device_options);
252 struct gve_device_option *dev_opt;
253 int i;
254
255 /* The options struct directly follows the device descriptor. */
256 dev_opt = (void *)(descriptor + 1);
257 for (i = 0; i < num_options; i++) {
258 struct gve_device_option *next_opt;
259
260 next_opt = gve_get_next_option(descriptor, dev_opt);
261 if (!next_opt) {
262 dev_err(&priv->dev->dev,
263 "options exceed device_descriptor's total length.\n");
264 return -EINVAL;
265 }
266
267 gve_parse_device_option(priv, descriptor, dev_opt,
268 dev_op_gqi_rda, dev_op_gqi_qpl,
269 dev_op_dqo_rda, dev_op_jumbo_frames,
270 dev_op_dqo_qpl, dev_op_buffer_sizes,
271 dev_op_flow_steering, dev_op_rss_config,
272 dev_op_modify_ring);
273 dev_opt = next_opt;
274 }
275
276 return 0;
277}
278
279int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
280{
281 priv->adminq_pool = dma_pool_create("adminq_pool", dev,
282 GVE_ADMINQ_BUFFER_SIZE, 0, 0);
283 if (unlikely(!priv->adminq_pool))
284 return -ENOMEM;
285 priv->adminq = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL,
286 &priv->adminq_bus_addr);
287 if (unlikely(!priv->adminq)) {
288 dma_pool_destroy(priv->adminq_pool);
289 return -ENOMEM;
290 }
291
292 priv->adminq_mask =
293 (GVE_ADMINQ_BUFFER_SIZE / sizeof(union gve_adminq_command)) - 1;
294 priv->adminq_prod_cnt = 0;
295 priv->adminq_cmd_fail = 0;
296 priv->adminq_timeouts = 0;
297 priv->adminq_describe_device_cnt = 0;
298 priv->adminq_cfg_device_resources_cnt = 0;
299 priv->adminq_register_page_list_cnt = 0;
300 priv->adminq_unregister_page_list_cnt = 0;
301 priv->adminq_create_tx_queue_cnt = 0;
302 priv->adminq_create_rx_queue_cnt = 0;
303 priv->adminq_destroy_tx_queue_cnt = 0;
304 priv->adminq_destroy_rx_queue_cnt = 0;
305 priv->adminq_dcfg_device_resources_cnt = 0;
306 priv->adminq_set_driver_parameter_cnt = 0;
307 priv->adminq_report_stats_cnt = 0;
308 priv->adminq_report_link_speed_cnt = 0;
309 priv->adminq_get_ptype_map_cnt = 0;
310 priv->adminq_query_flow_rules_cnt = 0;
311 priv->adminq_cfg_flow_rule_cnt = 0;
312 priv->adminq_cfg_rss_cnt = 0;
313 priv->adminq_query_rss_cnt = 0;
314
315 /* Setup Admin queue with the device */
316 if (priv->pdev->revision < 0x1) {
317 iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
318 &priv->reg_bar0->adminq_pfn);
319 } else {
320 iowrite16be(GVE_ADMINQ_BUFFER_SIZE,
321 &priv->reg_bar0->adminq_length);
322#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
323 iowrite32be(priv->adminq_bus_addr >> 32,
324 &priv->reg_bar0->adminq_base_address_hi);
325#endif
326 iowrite32be(priv->adminq_bus_addr,
327 &priv->reg_bar0->adminq_base_address_lo);
328 iowrite32be(GVE_DRIVER_STATUS_RUN_MASK, &priv->reg_bar0->driver_status);
329 }
330 mutex_init(&priv->adminq_lock);
331 gve_set_admin_queue_ok(priv);
332 return 0;
333}
334
335void gve_adminq_release(struct gve_priv *priv)
336{
337 int i = 0;
338
339 /* Tell the device the adminq is leaving */
340 if (priv->pdev->revision < 0x1) {
341 iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
342 while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
343 /* If this is reached the device is unrecoverable and still
344 * holding memory. Continue looping to avoid memory corruption,
345 * but WARN so it is visible what is going on.
346 */
347 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
348 WARN(1, "Unrecoverable platform error!");
349 i++;
350 msleep(GVE_ADMINQ_SLEEP_LEN);
351 }
352 } else {
353 iowrite32be(GVE_DRIVER_STATUS_RESET_MASK, &priv->reg_bar0->driver_status);
354 while (!(ioread32be(&priv->reg_bar0->device_status)
355 & GVE_DEVICE_STATUS_DEVICE_IS_RESET)) {
356 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
357 WARN(1, "Unrecoverable platform error!");
358 i++;
359 msleep(GVE_ADMINQ_SLEEP_LEN);
360 }
361 }
362 gve_clear_device_rings_ok(priv);
363 gve_clear_device_resources_ok(priv);
364 gve_clear_admin_queue_ok(priv);
365}
366
367void gve_adminq_free(struct device *dev, struct gve_priv *priv)
368{
369 if (!gve_get_admin_queue_ok(priv))
370 return;
371 gve_adminq_release(priv);
372 dma_pool_free(priv->adminq_pool, priv->adminq, priv->adminq_bus_addr);
373 dma_pool_destroy(priv->adminq_pool);
374 gve_clear_admin_queue_ok(priv);
375}
376
377static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
378{
379 iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
380}
381
382static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
383{
384 int i;
385
386 for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
387 if (ioread32be(&priv->reg_bar0->adminq_event_counter)
388 == prod_cnt)
389 return true;
390 msleep(GVE_ADMINQ_SLEEP_LEN);
391 }
392
393 return false;
394}
395
396static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
397{
398 if (status != GVE_ADMINQ_COMMAND_PASSED &&
399 status != GVE_ADMINQ_COMMAND_UNSET) {
400 dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status);
401 priv->adminq_cmd_fail++;
402 }
403 switch (status) {
404 case GVE_ADMINQ_COMMAND_PASSED:
405 return 0;
406 case GVE_ADMINQ_COMMAND_UNSET:
407 dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
408 return -EINVAL;
409 case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
410 case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
411 case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
412 case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
413 case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
414 return -EAGAIN;
415 case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
416 case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
417 case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
418 case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
419 case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
420 case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
421 return -EINVAL;
422 case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
423 return -ETIME;
424 case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
425 case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
426 return -EACCES;
427 case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
428 return -ENOMEM;
429 case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
430 return -EOPNOTSUPP;
431 default:
432 dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status);
433 return -EINVAL;
434 }
435}
436
437/* Flushes all AQ commands currently queued and waits for them to complete.
438 * If there are failures, it will return the first error.
439 */
440static int gve_adminq_kick_and_wait(struct gve_priv *priv)
441{
442 int tail, head;
443 int i;
444
445 tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
446 head = priv->adminq_prod_cnt;
447
448 gve_adminq_kick_cmd(priv, head);
449 if (!gve_adminq_wait_for_cmd(priv, head)) {
450 dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n");
451 priv->adminq_timeouts++;
452 return -ENOTRECOVERABLE;
453 }
454
455 for (i = tail; i < head; i++) {
456 union gve_adminq_command *cmd;
457 u32 status, err;
458
459 cmd = &priv->adminq[i & priv->adminq_mask];
460 status = be32_to_cpu(READ_ONCE(cmd->status));
461 err = gve_adminq_parse_err(priv, status);
462 if (err)
463 // Return the first error if we failed.
464 return err;
465 }
466
467 return 0;
468}
469
470/* This function is not threadsafe - the caller is responsible for any
471 * necessary locks.
472 */
473static int gve_adminq_issue_cmd(struct gve_priv *priv,
474 union gve_adminq_command *cmd_orig)
475{
476 union gve_adminq_command *cmd;
477 u32 opcode;
478 u32 tail;
479
480 tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
481
482 // Check if next command will overflow the buffer.
483 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
484 (tail & priv->adminq_mask)) {
485 int err;
486
487 // Flush existing commands to make room.
488 err = gve_adminq_kick_and_wait(priv);
489 if (err)
490 return err;
491
492 // Retry.
493 tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
494 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
495 (tail & priv->adminq_mask)) {
496 // This should never happen. We just flushed the
497 // command queue so there should be enough space.
498 return -ENOMEM;
499 }
500 }
501
502 cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
503 priv->adminq_prod_cnt++;
504
505 memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
506 opcode = be32_to_cpu(READ_ONCE(cmd->opcode));
507 if (opcode == GVE_ADMINQ_EXTENDED_COMMAND)
508 opcode = be32_to_cpu(cmd->extended_command.inner_opcode);
509
510 switch (opcode) {
511 case GVE_ADMINQ_DESCRIBE_DEVICE:
512 priv->adminq_describe_device_cnt++;
513 break;
514 case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
515 priv->adminq_cfg_device_resources_cnt++;
516 break;
517 case GVE_ADMINQ_REGISTER_PAGE_LIST:
518 priv->adminq_register_page_list_cnt++;
519 break;
520 case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
521 priv->adminq_unregister_page_list_cnt++;
522 break;
523 case GVE_ADMINQ_CREATE_TX_QUEUE:
524 priv->adminq_create_tx_queue_cnt++;
525 break;
526 case GVE_ADMINQ_CREATE_RX_QUEUE:
527 priv->adminq_create_rx_queue_cnt++;
528 break;
529 case GVE_ADMINQ_DESTROY_TX_QUEUE:
530 priv->adminq_destroy_tx_queue_cnt++;
531 break;
532 case GVE_ADMINQ_DESTROY_RX_QUEUE:
533 priv->adminq_destroy_rx_queue_cnt++;
534 break;
535 case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
536 priv->adminq_dcfg_device_resources_cnt++;
537 break;
538 case GVE_ADMINQ_SET_DRIVER_PARAMETER:
539 priv->adminq_set_driver_parameter_cnt++;
540 break;
541 case GVE_ADMINQ_REPORT_STATS:
542 priv->adminq_report_stats_cnt++;
543 break;
544 case GVE_ADMINQ_REPORT_LINK_SPEED:
545 priv->adminq_report_link_speed_cnt++;
546 break;
547 case GVE_ADMINQ_GET_PTYPE_MAP:
548 priv->adminq_get_ptype_map_cnt++;
549 break;
550 case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY:
551 priv->adminq_verify_driver_compatibility_cnt++;
552 break;
553 case GVE_ADMINQ_QUERY_FLOW_RULES:
554 priv->adminq_query_flow_rules_cnt++;
555 break;
556 case GVE_ADMINQ_CONFIGURE_FLOW_RULE:
557 priv->adminq_cfg_flow_rule_cnt++;
558 break;
559 case GVE_ADMINQ_CONFIGURE_RSS:
560 priv->adminq_cfg_rss_cnt++;
561 break;
562 case GVE_ADMINQ_QUERY_RSS:
563 priv->adminq_query_rss_cnt++;
564 break;
565 default:
566 dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode);
567 }
568
569 return 0;
570}
571
572static int gve_adminq_execute_cmd(struct gve_priv *priv,
573 union gve_adminq_command *cmd_orig)
574{
575 u32 tail, head;
576 int err;
577
578 mutex_lock(&priv->adminq_lock);
579 tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
580 head = priv->adminq_prod_cnt;
581 if (tail != head) {
582 err = -EINVAL;
583 goto out;
584 }
585
586 err = gve_adminq_issue_cmd(priv, cmd_orig);
587 if (err)
588 goto out;
589
590 err = gve_adminq_kick_and_wait(priv);
591
592out:
593 mutex_unlock(&priv->adminq_lock);
594 return err;
595}
596
597static int gve_adminq_execute_extended_cmd(struct gve_priv *priv, u32 opcode,
598 size_t cmd_size, void *cmd_orig)
599{
600 union gve_adminq_command cmd;
601 dma_addr_t inner_cmd_bus;
602 void *inner_cmd;
603 int err;
604
605 inner_cmd = dma_alloc_coherent(&priv->pdev->dev, cmd_size,
606 &inner_cmd_bus, GFP_KERNEL);
607 if (!inner_cmd)
608 return -ENOMEM;
609
610 memcpy(inner_cmd, cmd_orig, cmd_size);
611
612 memset(&cmd, 0, sizeof(cmd));
613 cmd.opcode = cpu_to_be32(GVE_ADMINQ_EXTENDED_COMMAND);
614 cmd.extended_command = (struct gve_adminq_extended_command) {
615 .inner_opcode = cpu_to_be32(opcode),
616 .inner_length = cpu_to_be32(cmd_size),
617 .inner_command_addr = cpu_to_be64(inner_cmd_bus),
618 };
619
620 err = gve_adminq_execute_cmd(priv, &cmd);
621
622 dma_free_coherent(&priv->pdev->dev, cmd_size, inner_cmd, inner_cmd_bus);
623 return err;
624}
625
626/* The device specifies that the management vector can either be the first irq
627 * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
628 * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
629 * the management vector is first.
630 *
631 * gve arranges the msix vectors so that the management vector is last.
632 */
633#define GVE_NTFY_BLK_BASE_MSIX_IDX 0
634int gve_adminq_configure_device_resources(struct gve_priv *priv,
635 dma_addr_t counter_array_bus_addr,
636 u32 num_counters,
637 dma_addr_t db_array_bus_addr,
638 u32 num_ntfy_blks)
639{
640 union gve_adminq_command cmd;
641
642 memset(&cmd, 0, sizeof(cmd));
643 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
644 cmd.configure_device_resources =
645 (struct gve_adminq_configure_device_resources) {
646 .counter_array = cpu_to_be64(counter_array_bus_addr),
647 .num_counters = cpu_to_be32(num_counters),
648 .irq_db_addr = cpu_to_be64(db_array_bus_addr),
649 .num_irq_dbs = cpu_to_be32(num_ntfy_blks),
650 .irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)),
651 .ntfy_blk_msix_base_idx =
652 cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
653 .queue_format = priv->queue_format,
654 };
655
656 return gve_adminq_execute_cmd(priv, &cmd);
657}
658
659int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
660{
661 union gve_adminq_command cmd;
662
663 memset(&cmd, 0, sizeof(cmd));
664 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
665
666 return gve_adminq_execute_cmd(priv, &cmd);
667}
668
669static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
670{
671 struct gve_tx_ring *tx = &priv->tx[queue_index];
672 union gve_adminq_command cmd;
673
674 memset(&cmd, 0, sizeof(cmd));
675 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
676 cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
677 .queue_id = cpu_to_be32(queue_index),
678 .queue_resources_addr =
679 cpu_to_be64(tx->q_resources_bus),
680 .tx_ring_addr = cpu_to_be64(tx->bus),
681 .ntfy_id = cpu_to_be32(tx->ntfy_id),
682 .tx_ring_size = cpu_to_be16(priv->tx_desc_cnt),
683 };
684
685 if (gve_is_gqi(priv)) {
686 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
687 GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id;
688
689 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
690 } else {
691 u32 qpl_id = 0;
692
693 if (priv->queue_format == GVE_DQO_RDA_FORMAT)
694 qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
695 else
696 qpl_id = tx->dqo.qpl->id;
697 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
698 cmd.create_tx_queue.tx_comp_ring_addr =
699 cpu_to_be64(tx->complq_bus_dqo);
700 cmd.create_tx_queue.tx_comp_ring_size =
701 cpu_to_be16(priv->tx_desc_cnt);
702 }
703
704 return gve_adminq_issue_cmd(priv, &cmd);
705}
706
707int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues)
708{
709 int err;
710 int i;
711
712 for (i = start_id; i < start_id + num_queues; i++) {
713 err = gve_adminq_create_tx_queue(priv, i);
714 if (err)
715 return err;
716 }
717
718 return gve_adminq_kick_and_wait(priv);
719}
720
721static void gve_adminq_get_create_rx_queue_cmd(struct gve_priv *priv,
722 union gve_adminq_command *cmd,
723 u32 queue_index)
724{
725 struct gve_rx_ring *rx = &priv->rx[queue_index];
726
727 memset(cmd, 0, sizeof(*cmd));
728 cmd->opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
729 cmd->create_rx_queue = (struct gve_adminq_create_rx_queue) {
730 .queue_id = cpu_to_be32(queue_index),
731 .ntfy_id = cpu_to_be32(rx->ntfy_id),
732 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
733 .rx_ring_size = cpu_to_be16(priv->rx_desc_cnt),
734 };
735
736 if (gve_is_gqi(priv)) {
737 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
738 GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id;
739
740 cmd->create_rx_queue.rx_desc_ring_addr =
741 cpu_to_be64(rx->desc.bus);
742 cmd->create_rx_queue.rx_data_ring_addr =
743 cpu_to_be64(rx->data.data_bus);
744 cmd->create_rx_queue.index = cpu_to_be32(queue_index);
745 cmd->create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
746 cmd->create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size);
747 } else {
748 u32 qpl_id = 0;
749
750 if (priv->queue_format == GVE_DQO_RDA_FORMAT)
751 qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
752 else
753 qpl_id = rx->dqo.qpl->id;
754 cmd->create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
755 cmd->create_rx_queue.rx_desc_ring_addr =
756 cpu_to_be64(rx->dqo.complq.bus);
757 cmd->create_rx_queue.rx_data_ring_addr =
758 cpu_to_be64(rx->dqo.bufq.bus);
759 cmd->create_rx_queue.packet_buffer_size =
760 cpu_to_be16(priv->data_buffer_size_dqo);
761 cmd->create_rx_queue.rx_buff_ring_size =
762 cpu_to_be16(priv->rx_desc_cnt);
763 cmd->create_rx_queue.enable_rsc =
764 !!(priv->dev->features & NETIF_F_LRO);
765 if (priv->header_split_enabled)
766 cmd->create_rx_queue.header_buffer_size =
767 cpu_to_be16(priv->header_buf_size);
768 }
769}
770
771static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
772{
773 union gve_adminq_command cmd;
774
775 gve_adminq_get_create_rx_queue_cmd(priv, &cmd, queue_index);
776 return gve_adminq_issue_cmd(priv, &cmd);
777}
778
779/* Unlike gve_adminq_create_rx_queue, this actually rings the doorbell */
780int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index)
781{
782 union gve_adminq_command cmd;
783
784 gve_adminq_get_create_rx_queue_cmd(priv, &cmd, queue_index);
785 return gve_adminq_execute_cmd(priv, &cmd);
786}
787
788int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues)
789{
790 int err;
791 int i;
792
793 for (i = 0; i < num_queues; i++) {
794 err = gve_adminq_create_rx_queue(priv, i);
795 if (err)
796 return err;
797 }
798
799 return gve_adminq_kick_and_wait(priv);
800}
801
802static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
803{
804 union gve_adminq_command cmd;
805 int err;
806
807 memset(&cmd, 0, sizeof(cmd));
808 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
809 cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
810 .queue_id = cpu_to_be32(queue_index),
811 };
812
813 err = gve_adminq_issue_cmd(priv, &cmd);
814 if (err)
815 return err;
816
817 return 0;
818}
819
820int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues)
821{
822 int err;
823 int i;
824
825 for (i = start_id; i < start_id + num_queues; i++) {
826 err = gve_adminq_destroy_tx_queue(priv, i);
827 if (err)
828 return err;
829 }
830
831 return gve_adminq_kick_and_wait(priv);
832}
833
834static void gve_adminq_make_destroy_rx_queue_cmd(union gve_adminq_command *cmd,
835 u32 queue_index)
836{
837 memset(cmd, 0, sizeof(*cmd));
838 cmd->opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
839 cmd->destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
840 .queue_id = cpu_to_be32(queue_index),
841 };
842}
843
844static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
845{
846 union gve_adminq_command cmd;
847
848 gve_adminq_make_destroy_rx_queue_cmd(&cmd, queue_index);
849 return gve_adminq_issue_cmd(priv, &cmd);
850}
851
852/* Unlike gve_adminq_destroy_rx_queue, this actually rings the doorbell */
853int gve_adminq_destroy_single_rx_queue(struct gve_priv *priv, u32 queue_index)
854{
855 union gve_adminq_command cmd;
856
857 gve_adminq_make_destroy_rx_queue_cmd(&cmd, queue_index);
858 return gve_adminq_execute_cmd(priv, &cmd);
859}
860
861int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
862{
863 int err;
864 int i;
865
866 for (i = 0; i < num_queues; i++) {
867 err = gve_adminq_destroy_rx_queue(priv, i);
868 if (err)
869 return err;
870 }
871
872 return gve_adminq_kick_and_wait(priv);
873}
874
875static void gve_set_default_desc_cnt(struct gve_priv *priv,
876 const struct gve_device_descriptor *descriptor)
877{
878 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
879 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
880
881 /* set default ranges */
882 priv->max_tx_desc_cnt = priv->tx_desc_cnt;
883 priv->max_rx_desc_cnt = priv->rx_desc_cnt;
884 priv->min_tx_desc_cnt = priv->tx_desc_cnt;
885 priv->min_rx_desc_cnt = priv->rx_desc_cnt;
886}
887
888static void gve_enable_supported_features(struct gve_priv *priv,
889 u32 supported_features_mask,
890 const struct gve_device_option_jumbo_frames
891 *dev_op_jumbo_frames,
892 const struct gve_device_option_dqo_qpl
893 *dev_op_dqo_qpl,
894 const struct gve_device_option_buffer_sizes
895 *dev_op_buffer_sizes,
896 const struct gve_device_option_flow_steering
897 *dev_op_flow_steering,
898 const struct gve_device_option_rss_config
899 *dev_op_rss_config,
900 const struct gve_device_option_modify_ring
901 *dev_op_modify_ring)
902{
903 /* Before control reaches this point, the page-size-capped max MTU from
904 * the gve_device_descriptor field has already been stored in
905 * priv->dev->max_mtu. We overwrite it with the true max MTU below.
906 */
907 if (dev_op_jumbo_frames &&
908 (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
909 dev_info(&priv->pdev->dev,
910 "JUMBO FRAMES device option enabled.\n");
911 priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
912 }
913
914 /* Override pages for qpl for DQO-QPL */
915 if (dev_op_dqo_qpl) {
916 priv->tx_pages_per_qpl =
917 be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl);
918 if (priv->tx_pages_per_qpl == 0)
919 priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES;
920 }
921
922 if (dev_op_buffer_sizes &&
923 (supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) {
924 priv->max_rx_buffer_size =
925 be16_to_cpu(dev_op_buffer_sizes->packet_buffer_size);
926 priv->header_buf_size =
927 be16_to_cpu(dev_op_buffer_sizes->header_buffer_size);
928 dev_info(&priv->pdev->dev,
929 "BUFFER SIZES device option enabled with max_rx_buffer_size of %u, header_buf_size of %u.\n",
930 priv->max_rx_buffer_size, priv->header_buf_size);
931 }
932
933 /* Read and store ring size ranges given by device */
934 if (dev_op_modify_ring &&
935 (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) {
936 priv->modify_ring_size_enabled = true;
937
938 /* max ring size for DQO QPL should not be overwritten because of device limit */
939 if (priv->queue_format != GVE_DQO_QPL_FORMAT) {
940 priv->max_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_rx_ring_size);
941 priv->max_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_tx_ring_size);
942 }
943 if (priv->default_min_ring_size) {
944 /* If device hasn't provided minimums, use default minimums */
945 priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;
946 priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;
947 } else {
948 priv->min_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_rx_ring_size);
949 priv->min_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->min_tx_ring_size);
950 }
951 }
952
953 if (dev_op_flow_steering &&
954 (supported_features_mask & GVE_SUP_FLOW_STEERING_MASK)) {
955 if (dev_op_flow_steering->max_flow_rules) {
956 priv->max_flow_rules =
957 be32_to_cpu(dev_op_flow_steering->max_flow_rules);
958 priv->dev->hw_features |= NETIF_F_NTUPLE;
959 dev_info(&priv->pdev->dev,
960 "FLOW STEERING device option enabled with max rule limit of %u.\n",
961 priv->max_flow_rules);
962 }
963 }
964
965 if (dev_op_rss_config &&
966 (supported_features_mask & GVE_SUP_RSS_CONFIG_MASK)) {
967 priv->rss_key_size =
968 be16_to_cpu(dev_op_rss_config->hash_key_size);
969 priv->rss_lut_size =
970 be16_to_cpu(dev_op_rss_config->hash_lut_size);
971 }
972}
973
974int gve_adminq_describe_device(struct gve_priv *priv)
975{
976 struct gve_device_option_flow_steering *dev_op_flow_steering = NULL;
977 struct gve_device_option_buffer_sizes *dev_op_buffer_sizes = NULL;
978 struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
979 struct gve_device_option_modify_ring *dev_op_modify_ring = NULL;
980 struct gve_device_option_rss_config *dev_op_rss_config = NULL;
981 struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
982 struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
983 struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
984 struct gve_device_option_dqo_qpl *dev_op_dqo_qpl = NULL;
985 struct gve_device_descriptor *descriptor;
986 u32 supported_features_mask = 0;
987 union gve_adminq_command cmd;
988 dma_addr_t descriptor_bus;
989 int err = 0;
990 u8 *mac;
991 u16 mtu;
992
993 memset(&cmd, 0, sizeof(cmd));
994 descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL,
995 &descriptor_bus);
996 if (!descriptor)
997 return -ENOMEM;
998 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
999 cmd.describe_device.device_descriptor_addr =
1000 cpu_to_be64(descriptor_bus);
1001 cmd.describe_device.device_descriptor_version =
1002 cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
1003 cmd.describe_device.available_length =
1004 cpu_to_be32(GVE_ADMINQ_BUFFER_SIZE);
1005
1006 err = gve_adminq_execute_cmd(priv, &cmd);
1007 if (err)
1008 goto free_device_descriptor;
1009
1010 err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda,
1011 &dev_op_gqi_qpl, &dev_op_dqo_rda,
1012 &dev_op_jumbo_frames, &dev_op_dqo_qpl,
1013 &dev_op_buffer_sizes,
1014 &dev_op_flow_steering,
1015 &dev_op_rss_config,
1016 &dev_op_modify_ring);
1017 if (err)
1018 goto free_device_descriptor;
1019
1020 /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
1021 * is not set to GqiRda, choose the queue format in a priority order:
1022 * DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default.
1023 */
1024 if (dev_op_dqo_rda) {
1025 priv->queue_format = GVE_DQO_RDA_FORMAT;
1026 dev_info(&priv->pdev->dev,
1027 "Driver is running with DQO RDA queue format.\n");
1028 supported_features_mask =
1029 be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
1030 } else if (dev_op_dqo_qpl) {
1031 priv->queue_format = GVE_DQO_QPL_FORMAT;
1032 supported_features_mask =
1033 be32_to_cpu(dev_op_dqo_qpl->supported_features_mask);
1034 } else if (dev_op_gqi_rda) {
1035 priv->queue_format = GVE_GQI_RDA_FORMAT;
1036 dev_info(&priv->pdev->dev,
1037 "Driver is running with GQI RDA queue format.\n");
1038 supported_features_mask =
1039 be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
1040 } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
1041 dev_info(&priv->pdev->dev,
1042 "Driver is running with GQI RDA queue format.\n");
1043 } else {
1044 priv->queue_format = GVE_GQI_QPL_FORMAT;
1045 if (dev_op_gqi_qpl)
1046 supported_features_mask =
1047 be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
1048 dev_info(&priv->pdev->dev,
1049 "Driver is running with GQI QPL queue format.\n");
1050 }
1051
1052 /* set default descriptor counts */
1053 gve_set_default_desc_cnt(priv, descriptor);
1054
1055 /* DQO supports LRO. */
1056 if (!gve_is_gqi(priv))
1057 priv->dev->hw_features |= NETIF_F_LRO;
1058
1059 priv->max_registered_pages =
1060 be64_to_cpu(descriptor->max_registered_pages);
1061 mtu = be16_to_cpu(descriptor->mtu);
1062 if (mtu < ETH_MIN_MTU) {
1063 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
1064 err = -EINVAL;
1065 goto free_device_descriptor;
1066 }
1067 priv->dev->max_mtu = mtu;
1068 priv->num_event_counters = be16_to_cpu(descriptor->counters);
1069 eth_hw_addr_set(priv->dev, descriptor->mac);
1070 mac = descriptor->mac;
1071 dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
1072 priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
1073 priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
1074
1075 gve_enable_supported_features(priv, supported_features_mask,
1076 dev_op_jumbo_frames, dev_op_dqo_qpl,
1077 dev_op_buffer_sizes, dev_op_flow_steering,
1078 dev_op_rss_config, dev_op_modify_ring);
1079
1080free_device_descriptor:
1081 dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus);
1082 return err;
1083}
1084
1085int gve_adminq_register_page_list(struct gve_priv *priv,
1086 struct gve_queue_page_list *qpl)
1087{
1088 struct device *hdev = &priv->pdev->dev;
1089 u32 num_entries = qpl->num_entries;
1090 u32 size = num_entries * sizeof(qpl->page_buses[0]);
1091 union gve_adminq_command cmd;
1092 dma_addr_t page_list_bus;
1093 __be64 *page_list;
1094 int err;
1095 int i;
1096
1097 memset(&cmd, 0, sizeof(cmd));
1098 page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
1099 if (!page_list)
1100 return -ENOMEM;
1101
1102 for (i = 0; i < num_entries; i++)
1103 page_list[i] = cpu_to_be64(qpl->page_buses[i]);
1104
1105 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
1106 cmd.reg_page_list = (struct gve_adminq_register_page_list) {
1107 .page_list_id = cpu_to_be32(qpl->id),
1108 .num_pages = cpu_to_be32(num_entries),
1109 .page_address_list_addr = cpu_to_be64(page_list_bus),
1110 .page_size = cpu_to_be64(PAGE_SIZE),
1111 };
1112
1113 err = gve_adminq_execute_cmd(priv, &cmd);
1114 dma_free_coherent(hdev, size, page_list, page_list_bus);
1115 return err;
1116}
1117
1118int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
1119{
1120 union gve_adminq_command cmd;
1121
1122 memset(&cmd, 0, sizeof(cmd));
1123 cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
1124 cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
1125 .page_list_id = cpu_to_be32(page_list_id),
1126 };
1127
1128 return gve_adminq_execute_cmd(priv, &cmd);
1129}
1130
1131int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
1132{
1133 union gve_adminq_command cmd;
1134
1135 memset(&cmd, 0, sizeof(cmd));
1136 cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
1137 cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
1138 .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
1139 .parameter_value = cpu_to_be64(mtu),
1140 };
1141
1142 return gve_adminq_execute_cmd(priv, &cmd);
1143}
1144
1145int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
1146 dma_addr_t stats_report_addr, u64 interval)
1147{
1148 union gve_adminq_command cmd;
1149
1150 memset(&cmd, 0, sizeof(cmd));
1151 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS);
1152 cmd.report_stats = (struct gve_adminq_report_stats) {
1153 .stats_report_len = cpu_to_be64(stats_report_len),
1154 .stats_report_addr = cpu_to_be64(stats_report_addr),
1155 .interval = cpu_to_be64(interval),
1156 };
1157
1158 return gve_adminq_execute_cmd(priv, &cmd);
1159}
1160
1161int gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
1162 u64 driver_info_len,
1163 dma_addr_t driver_info_addr)
1164{
1165 union gve_adminq_command cmd;
1166
1167 memset(&cmd, 0, sizeof(cmd));
1168 cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY);
1169 cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
1170 .driver_info_len = cpu_to_be64(driver_info_len),
1171 .driver_info_addr = cpu_to_be64(driver_info_addr),
1172 };
1173
1174 return gve_adminq_execute_cmd(priv, &cmd);
1175}
1176
1177int gve_adminq_report_link_speed(struct gve_priv *priv)
1178{
1179 union gve_adminq_command gvnic_cmd;
1180 dma_addr_t link_speed_region_bus;
1181 __be64 *link_speed_region;
1182 int err;
1183
1184 link_speed_region =
1185 dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region),
1186 &link_speed_region_bus, GFP_KERNEL);
1187
1188 if (!link_speed_region)
1189 return -ENOMEM;
1190
1191 memset(&gvnic_cmd, 0, sizeof(gvnic_cmd));
1192 gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED);
1193 gvnic_cmd.report_link_speed.link_speed_address =
1194 cpu_to_be64(link_speed_region_bus);
1195
1196 err = gve_adminq_execute_cmd(priv, &gvnic_cmd);
1197
1198 priv->link_speed = be64_to_cpu(*link_speed_region);
1199 dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region,
1200 link_speed_region_bus);
1201 return err;
1202}
1203
1204int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
1205 struct gve_ptype_lut *ptype_lut)
1206{
1207 struct gve_ptype_map *ptype_map;
1208 union gve_adminq_command cmd;
1209 dma_addr_t ptype_map_bus;
1210 int err = 0;
1211 int i;
1212
1213 memset(&cmd, 0, sizeof(cmd));
1214 ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map),
1215 &ptype_map_bus, GFP_KERNEL);
1216 if (!ptype_map)
1217 return -ENOMEM;
1218
1219 cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP);
1220 cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
1221 .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)),
1222 .ptype_map_addr = cpu_to_be64(ptype_map_bus),
1223 };
1224
1225 err = gve_adminq_execute_cmd(priv, &cmd);
1226 if (err)
1227 goto err;
1228
1229 /* Populate ptype_lut. */
1230 for (i = 0; i < GVE_NUM_PTYPES; i++) {
1231 ptype_lut->ptypes[i].l3_type =
1232 ptype_map->ptypes[i].l3_type;
1233 ptype_lut->ptypes[i].l4_type =
1234 ptype_map->ptypes[i].l4_type;
1235 }
1236err:
1237 dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map,
1238 ptype_map_bus);
1239 return err;
1240}
1241
1242static int
1243gve_adminq_configure_flow_rule(struct gve_priv *priv,
1244 struct gve_adminq_configure_flow_rule *flow_rule_cmd)
1245{
1246 int err = gve_adminq_execute_extended_cmd(priv,
1247 GVE_ADMINQ_CONFIGURE_FLOW_RULE,
1248 sizeof(struct gve_adminq_configure_flow_rule),
1249 flow_rule_cmd);
1250
1251 if (err == -ETIME) {
1252 dev_err(&priv->pdev->dev, "Timeout to configure the flow rule, trigger reset");
1253 gve_reset(priv, true);
1254 } else if (!err) {
1255 priv->flow_rules_cache.rules_cache_synced = false;
1256 }
1257
1258 return err;
1259}
1260
1261int gve_adminq_add_flow_rule(struct gve_priv *priv, struct gve_adminq_flow_rule *rule, u32 loc)
1262{
1263 struct gve_adminq_configure_flow_rule flow_rule_cmd = {
1264 .opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_ADD),
1265 .location = cpu_to_be32(loc),
1266 .rule = *rule,
1267 };
1268
1269 return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd);
1270}
1271
1272int gve_adminq_del_flow_rule(struct gve_priv *priv, u32 loc)
1273{
1274 struct gve_adminq_configure_flow_rule flow_rule_cmd = {
1275 .opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_DEL),
1276 .location = cpu_to_be32(loc),
1277 };
1278
1279 return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd);
1280}
1281
1282int gve_adminq_reset_flow_rules(struct gve_priv *priv)
1283{
1284 struct gve_adminq_configure_flow_rule flow_rule_cmd = {
1285 .opcode = cpu_to_be16(GVE_FLOW_RULE_CFG_RESET),
1286 };
1287
1288 return gve_adminq_configure_flow_rule(priv, &flow_rule_cmd);
1289}
1290
1291int gve_adminq_configure_rss(struct gve_priv *priv, struct ethtool_rxfh_param *rxfh)
1292{
1293 dma_addr_t lut_bus = 0, key_bus = 0;
1294 u16 key_size = 0, lut_size = 0;
1295 union gve_adminq_command cmd;
1296 __be32 *lut = NULL;
1297 u8 hash_alg = 0;
1298 u8 *key = NULL;
1299 int err = 0;
1300 u16 i;
1301
1302 switch (rxfh->hfunc) {
1303 case ETH_RSS_HASH_NO_CHANGE:
1304 break;
1305 case ETH_RSS_HASH_TOP:
1306 hash_alg = ETH_RSS_HASH_TOP;
1307 break;
1308 default:
1309 return -EOPNOTSUPP;
1310 }
1311
1312 if (rxfh->indir) {
1313 lut_size = priv->rss_lut_size;
1314 lut = dma_alloc_coherent(&priv->pdev->dev,
1315 lut_size * sizeof(*lut),
1316 &lut_bus, GFP_KERNEL);
1317 if (!lut)
1318 return -ENOMEM;
1319
1320 for (i = 0; i < priv->rss_lut_size; i++)
1321 lut[i] = cpu_to_be32(rxfh->indir[i]);
1322 }
1323
1324 if (rxfh->key) {
1325 key_size = priv->rss_key_size;
1326 key = dma_alloc_coherent(&priv->pdev->dev,
1327 key_size, &key_bus, GFP_KERNEL);
1328 if (!key) {
1329 err = -ENOMEM;
1330 goto out;
1331 }
1332
1333 memcpy(key, rxfh->key, key_size);
1334 }
1335
1336 /* Zero-valued fields in the cmd.configure_rss instruct the device to
1337 * not update those fields.
1338 */
1339 memset(&cmd, 0, sizeof(cmd));
1340 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_RSS);
1341 cmd.configure_rss = (struct gve_adminq_configure_rss) {
1342 .hash_types = cpu_to_be16(BIT(GVE_RSS_HASH_TCPV4) |
1343 BIT(GVE_RSS_HASH_UDPV4) |
1344 BIT(GVE_RSS_HASH_TCPV6) |
1345 BIT(GVE_RSS_HASH_UDPV6)),
1346 .hash_alg = hash_alg,
1347 .hash_key_size = cpu_to_be16(key_size),
1348 .hash_lut_size = cpu_to_be16(lut_size),
1349 .hash_key_addr = cpu_to_be64(key_bus),
1350 .hash_lut_addr = cpu_to_be64(lut_bus),
1351 };
1352
1353 err = gve_adminq_execute_cmd(priv, &cmd);
1354
1355out:
1356 if (lut)
1357 dma_free_coherent(&priv->pdev->dev,
1358 lut_size * sizeof(*lut),
1359 lut, lut_bus);
1360 if (key)
1361 dma_free_coherent(&priv->pdev->dev,
1362 key_size, key, key_bus);
1363 return err;
1364}
1365
1366/* In the dma memory that the driver allocated for the device to query the flow rules, the device
1367 * will first write it with a struct of gve_query_flow_rules_descriptor. Next to it, the device
1368 * will write an array of rules or rule ids with the count that specified in the descriptor.
1369 * For GVE_FLOW_RULE_QUERY_STATS, the device will only write the descriptor.
1370 */
1371static int gve_adminq_process_flow_rules_query(struct gve_priv *priv, u16 query_opcode,
1372 struct gve_query_flow_rules_descriptor *descriptor)
1373{
1374 struct gve_flow_rules_cache *flow_rules_cache = &priv->flow_rules_cache;
1375 u32 num_queried_rules, total_memory_len, rule_info_len;
1376 void *rule_info;
1377
1378 total_memory_len = be32_to_cpu(descriptor->total_length);
1379 num_queried_rules = be32_to_cpu(descriptor->num_queried_rules);
1380 rule_info = (void *)(descriptor + 1);
1381
1382 switch (query_opcode) {
1383 case GVE_FLOW_RULE_QUERY_RULES:
1384 rule_info_len = num_queried_rules * sizeof(*flow_rules_cache->rules_cache);
1385 if (sizeof(*descriptor) + rule_info_len != total_memory_len) {
1386 dev_err(&priv->dev->dev, "flow rules query is out of memory.\n");
1387 return -ENOMEM;
1388 }
1389
1390 memcpy(flow_rules_cache->rules_cache, rule_info, rule_info_len);
1391 flow_rules_cache->rules_cache_num = num_queried_rules;
1392 break;
1393 case GVE_FLOW_RULE_QUERY_IDS:
1394 rule_info_len = num_queried_rules * sizeof(*flow_rules_cache->rule_ids_cache);
1395 if (sizeof(*descriptor) + rule_info_len != total_memory_len) {
1396 dev_err(&priv->dev->dev, "flow rule ids query is out of memory.\n");
1397 return -ENOMEM;
1398 }
1399
1400 memcpy(flow_rules_cache->rule_ids_cache, rule_info, rule_info_len);
1401 flow_rules_cache->rule_ids_cache_num = num_queried_rules;
1402 break;
1403 case GVE_FLOW_RULE_QUERY_STATS:
1404 priv->num_flow_rules = be32_to_cpu(descriptor->num_flow_rules);
1405 priv->max_flow_rules = be32_to_cpu(descriptor->max_flow_rules);
1406 return 0;
1407 default:
1408 return -EINVAL;
1409 }
1410
1411 return 0;
1412}
1413
1414int gve_adminq_query_flow_rules(struct gve_priv *priv, u16 query_opcode, u32 starting_loc)
1415{
1416 struct gve_query_flow_rules_descriptor *descriptor;
1417 union gve_adminq_command cmd;
1418 dma_addr_t descriptor_bus;
1419 int err = 0;
1420
1421 memset(&cmd, 0, sizeof(cmd));
1422 descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL, &descriptor_bus);
1423 if (!descriptor)
1424 return -ENOMEM;
1425
1426 cmd.opcode = cpu_to_be32(GVE_ADMINQ_QUERY_FLOW_RULES);
1427 cmd.query_flow_rules = (struct gve_adminq_query_flow_rules) {
1428 .opcode = cpu_to_be16(query_opcode),
1429 .starting_rule_id = cpu_to_be32(starting_loc),
1430 .available_length = cpu_to_be64(GVE_ADMINQ_BUFFER_SIZE),
1431 .rule_descriptor_addr = cpu_to_be64(descriptor_bus),
1432 };
1433 err = gve_adminq_execute_cmd(priv, &cmd);
1434 if (err)
1435 goto out;
1436
1437 err = gve_adminq_process_flow_rules_query(priv, query_opcode, descriptor);
1438
1439out:
1440 dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus);
1441 return err;
1442}
1443
1444static int gve_adminq_process_rss_query(struct gve_priv *priv,
1445 struct gve_query_rss_descriptor *descriptor,
1446 struct ethtool_rxfh_param *rxfh)
1447{
1448 u32 total_memory_length;
1449 u16 hash_lut_length;
1450 void *rss_info_addr;
1451 __be32 *lut;
1452 u16 i;
1453
1454 total_memory_length = be32_to_cpu(descriptor->total_length);
1455 hash_lut_length = priv->rss_lut_size * sizeof(*rxfh->indir);
1456
1457 if (sizeof(*descriptor) + priv->rss_key_size + hash_lut_length != total_memory_length) {
1458 dev_err(&priv->dev->dev,
1459 "rss query desc from device has invalid length parameter.\n");
1460 return -EINVAL;
1461 }
1462
1463 rxfh->hfunc = descriptor->hash_alg;
1464
1465 rss_info_addr = (void *)(descriptor + 1);
1466 if (rxfh->key)
1467 memcpy(rxfh->key, rss_info_addr, priv->rss_key_size);
1468
1469 rss_info_addr += priv->rss_key_size;
1470 lut = (__be32 *)rss_info_addr;
1471 if (rxfh->indir) {
1472 for (i = 0; i < priv->rss_lut_size; i++)
1473 rxfh->indir[i] = be32_to_cpu(lut[i]);
1474 }
1475
1476 return 0;
1477}
1478
1479int gve_adminq_query_rss_config(struct gve_priv *priv, struct ethtool_rxfh_param *rxfh)
1480{
1481 struct gve_query_rss_descriptor *descriptor;
1482 union gve_adminq_command cmd;
1483 dma_addr_t descriptor_bus;
1484 int err = 0;
1485
1486 descriptor = dma_pool_alloc(priv->adminq_pool, GFP_KERNEL, &descriptor_bus);
1487 if (!descriptor)
1488 return -ENOMEM;
1489
1490 memset(&cmd, 0, sizeof(cmd));
1491 cmd.opcode = cpu_to_be32(GVE_ADMINQ_QUERY_RSS);
1492 cmd.query_rss = (struct gve_adminq_query_rss) {
1493 .available_length = cpu_to_be64(GVE_ADMINQ_BUFFER_SIZE),
1494 .rss_descriptor_addr = cpu_to_be64(descriptor_bus),
1495 };
1496 err = gve_adminq_execute_cmd(priv, &cmd);
1497 if (err)
1498 goto out;
1499
1500 err = gve_adminq_process_rss_query(priv, descriptor, rxfh);
1501
1502out:
1503 dma_pool_free(priv->adminq_pool, descriptor, descriptor_bus);
1504 return err;
1505}