Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2/* Marvell Octeon EP (EndPoint) Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8#include <linux/pci.h>
9#include <linux/etherdevice.h>
10#include <linux/vmalloc.h>
11
12#include "octep_config.h"
13#include "octep_main.h"
14
15static void octep_oq_reset_indices(struct octep_oq *oq)
16{
17 oq->host_read_idx = 0;
18 oq->host_refill_idx = 0;
19 oq->refill_count = 0;
20 oq->last_pkt_count = 0;
21 oq->pkts_pending = 0;
22}
23
24/**
25 * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.
26 *
27 * @oq: Octeon Rx queue data structure.
28 *
29 * Return: 0, if successfully filled receive buffers for all descriptors.
30 * -1, if failed to allocate a buffer or failed to map for DMA.
31 */
32static int octep_oq_fill_ring_buffers(struct octep_oq *oq)
33{
34 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
35 struct page *page;
36 u32 i;
37
38 for (i = 0; i < oq->max_count; i++) {
39 page = dev_alloc_page();
40 if (unlikely(!page)) {
41 dev_err(oq->dev, "Rx buffer alloc failed\n");
42 goto rx_buf_alloc_err;
43 }
44 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,
45 PAGE_SIZE,
46 DMA_FROM_DEVICE);
47 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {
48 dev_err(oq->dev,
49 "OQ-%d buffer alloc: DMA mapping error!\n",
50 oq->q_no);
51 put_page(page);
52 goto dma_map_err;
53 }
54 oq->buff_info[i].page = page;
55 }
56
57 return 0;
58
59dma_map_err:
60rx_buf_alloc_err:
61 while (i) {
62 i--;
63 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);
64 put_page(oq->buff_info[i].page);
65 oq->buff_info[i].page = NULL;
66 }
67
68 return -1;
69}
70
71/**
72 * octep_oq_refill() - refill buffers for used Rx ring descriptors.
73 *
74 * @oct: Octeon device private data structure.
75 * @oq: Octeon Rx queue data structure.
76 *
77 * Return: number of descriptors successfully refilled with receive buffers.
78 */
79static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq)
80{
81 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
82 struct page *page;
83 u32 refill_idx, i;
84
85 refill_idx = oq->host_refill_idx;
86 for (i = 0; i < oq->refill_count; i++) {
87 page = dev_alloc_page();
88 if (unlikely(!page)) {
89 dev_err(oq->dev, "refill: rx buffer alloc failed\n");
90 oq->stats->alloc_failures++;
91 break;
92 }
93
94 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
95 PAGE_SIZE, DMA_FROM_DEVICE);
96 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
97 dev_err(oq->dev,
98 "OQ-%d buffer refill: DMA mapping error!\n",
99 oq->q_no);
100 put_page(page);
101 oq->stats->alloc_failures++;
102 break;
103 }
104 oq->buff_info[refill_idx].page = page;
105 refill_idx++;
106 if (refill_idx == oq->max_count)
107 refill_idx = 0;
108 }
109 oq->host_refill_idx = refill_idx;
110 oq->refill_count -= i;
111
112 return i;
113}
114
115/**
116 * octep_setup_oq() - Setup a Rx queue.
117 *
118 * @oct: Octeon device private data structure.
119 * @q_no: Rx queue number to be setup.
120 *
121 * Allocate resources for a Rx queue.
122 */
123static int octep_setup_oq(struct octep_device *oct, int q_no)
124{
125 struct octep_oq *oq;
126 u32 desc_ring_size;
127
128 oq = vzalloc(sizeof(*oq));
129 if (!oq)
130 goto create_oq_fail;
131 oct->oq[q_no] = oq;
132
133 oq->octep_dev = oct;
134 oq->netdev = oct->netdev;
135 oq->dev = &oct->pdev->dev;
136 oq->q_no = q_no;
137 oq->stats = &oct->stats_oq[q_no];
138 oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);
139 oq->ring_size_mask = oq->max_count - 1;
140 oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);
141 oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE;
142
143 /* When the hardware/firmware supports additional capabilities,
144 * additional header is filled-in by Octeon after length field in
145 * Rx packets. this header contains additional packet information.
146 */
147 if (oct->conf->fw_info.rx_ol_flags)
148 oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE;
149
150 oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);
151
152 desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE;
153 oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,
154 &oq->desc_ring_dma, GFP_KERNEL);
155
156 if (unlikely(!oq->desc_ring)) {
157 dev_err(oq->dev,
158 "Failed to allocate DMA memory for OQ-%d !!\n", q_no);
159 goto desc_dma_alloc_err;
160 }
161
162 oq->buff_info = vcalloc(oq->max_count, OCTEP_OQ_RECVBUF_SIZE);
163 if (unlikely(!oq->buff_info)) {
164 dev_err(&oct->pdev->dev,
165 "Failed to allocate buffer info for OQ-%d\n", q_no);
166 goto buf_list_err;
167 }
168
169 if (octep_oq_fill_ring_buffers(oq))
170 goto oq_fill_buff_err;
171
172 octep_oq_reset_indices(oq);
173 oct->hw_ops.setup_oq_regs(oct, q_no);
174 oct->num_oqs++;
175
176 return 0;
177
178oq_fill_buff_err:
179 vfree(oq->buff_info);
180 oq->buff_info = NULL;
181buf_list_err:
182 dma_free_coherent(oq->dev, desc_ring_size,
183 oq->desc_ring, oq->desc_ring_dma);
184 oq->desc_ring = NULL;
185desc_dma_alloc_err:
186 vfree(oq);
187 oct->oq[q_no] = NULL;
188create_oq_fail:
189 return -1;
190}
191
192/**
193 * octep_oq_free_ring_buffers() - Free ring buffers.
194 *
195 * @oq: Octeon Rx queue data structure.
196 *
197 * Free receive buffers in unused Rx queue descriptors.
198 */
199static void octep_oq_free_ring_buffers(struct octep_oq *oq)
200{
201 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
202 int i;
203
204 if (!oq->desc_ring || !oq->buff_info)
205 return;
206
207 for (i = 0; i < oq->max_count; i++) {
208 if (oq->buff_info[i].page) {
209 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,
210 PAGE_SIZE, DMA_FROM_DEVICE);
211 put_page(oq->buff_info[i].page);
212 oq->buff_info[i].page = NULL;
213 desc_ring[i].buffer_ptr = 0;
214 }
215 }
216 octep_oq_reset_indices(oq);
217}
218
219/**
220 * octep_free_oq() - Free Rx queue resources.
221 *
222 * @oq: Octeon Rx queue data structure.
223 *
224 * Free all resources of a Rx queue.
225 */
226static int octep_free_oq(struct octep_oq *oq)
227{
228 struct octep_device *oct = oq->octep_dev;
229 int q_no = oq->q_no;
230
231 octep_oq_free_ring_buffers(oq);
232
233 vfree(oq->buff_info);
234
235 if (oq->desc_ring)
236 dma_free_coherent(oq->dev,
237 oq->max_count * OCTEP_OQ_DESC_SIZE,
238 oq->desc_ring, oq->desc_ring_dma);
239
240 vfree(oq);
241 oct->oq[q_no] = NULL;
242 oct->num_oqs--;
243 return 0;
244}
245
246/**
247 * octep_setup_oqs() - setup resources for all Rx queues.
248 *
249 * @oct: Octeon device private data structure.
250 */
251int octep_setup_oqs(struct octep_device *oct)
252{
253 int i, retval = 0;
254
255 oct->num_oqs = 0;
256 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
257 retval = octep_setup_oq(oct, i);
258 if (retval) {
259 dev_err(&oct->pdev->dev,
260 "Failed to setup OQ(RxQ)-%d.\n", i);
261 goto oq_setup_err;
262 }
263 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);
264 }
265
266 return 0;
267
268oq_setup_err:
269 while (i) {
270 i--;
271 octep_free_oq(oct->oq[i]);
272 }
273 return -1;
274}
275
276/**
277 * octep_oq_dbell_init() - Initialize Rx queue doorbell.
278 *
279 * @oct: Octeon device private data structure.
280 *
281 * Write number of descriptors to Rx queue doorbell register.
282 */
283void octep_oq_dbell_init(struct octep_device *oct)
284{
285 int i;
286
287 for (i = 0; i < oct->num_oqs; i++)
288 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
289}
290
291/**
292 * octep_free_oqs() - Free resources of all Rx queues.
293 *
294 * @oct: Octeon device private data structure.
295 */
296void octep_free_oqs(struct octep_device *oct)
297{
298 int i;
299
300 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
301 if (!oct->oq[i])
302 continue;
303 octep_free_oq(oct->oq[i]);
304 dev_dbg(&oct->pdev->dev,
305 "Successfully freed OQ(RxQ)-%d.\n", i);
306 }
307}
308
309/**
310 * octep_oq_check_hw_for_pkts() - Check for new Rx packets.
311 *
312 * @oct: Octeon device private data structure.
313 * @oq: Octeon Rx queue data structure.
314 *
315 * Return: packets received after previous check.
316 */
317static int octep_oq_check_hw_for_pkts(struct octep_device *oct,
318 struct octep_oq *oq)
319{
320 u32 pkt_count, new_pkts;
321
322 pkt_count = readl(oq->pkts_sent_reg);
323 new_pkts = pkt_count - oq->last_pkt_count;
324
325 /* Clear the hardware packets counter register if the rx queue is
326 * being processed continuously with-in a single interrupt and
327 * reached half its max value.
328 * this counter is not cleared every time read, to save write cycles.
329 */
330 if (unlikely(pkt_count > 0xF0000000U)) {
331 writel(pkt_count, oq->pkts_sent_reg);
332 pkt_count = readl(oq->pkts_sent_reg);
333 new_pkts += pkt_count;
334 }
335 oq->last_pkt_count = pkt_count;
336 oq->pkts_pending += new_pkts;
337 return new_pkts;
338}
339
340/**
341 * octep_oq_next_pkt() - Move to the next packet in Rx queue.
342 *
343 * @oq: Octeon Rx queue data structure.
344 * @buff_info: Current packet buffer info.
345 * @read_idx: Current packet index in the ring.
346 * @desc_used: Current packet descriptor number.
347 *
348 * Free the resources associated with a packet.
349 * Increment packet index in the ring and packet descriptor number.
350 */
351static void octep_oq_next_pkt(struct octep_oq *oq,
352 struct octep_rx_buffer *buff_info,
353 u32 *read_idx, u32 *desc_used)
354{
355 dma_unmap_page(oq->dev, oq->desc_ring[*read_idx].buffer_ptr,
356 PAGE_SIZE, DMA_FROM_DEVICE);
357 buff_info->page = NULL;
358 (*read_idx)++;
359 (*desc_used)++;
360 if (*read_idx == oq->max_count)
361 *read_idx = 0;
362}
363
364/**
365 * octep_oq_drop_rx() - Free the resources associated with a packet.
366 *
367 * @oq: Octeon Rx queue data structure.
368 * @buff_info: Current packet buffer info.
369 * @read_idx: Current packet index in the ring.
370 * @desc_used: Current packet descriptor number.
371 *
372 */
373static void octep_oq_drop_rx(struct octep_oq *oq,
374 struct octep_rx_buffer *buff_info,
375 u32 *read_idx, u32 *desc_used)
376{
377 int data_len = buff_info->len - oq->max_single_buffer_size;
378
379 while (data_len > 0) {
380 octep_oq_next_pkt(oq, buff_info, read_idx, desc_used);
381 data_len -= oq->buffer_size;
382 };
383}
384
385/**
386 * __octep_oq_process_rx() - Process hardware Rx queue and push to stack.
387 *
388 * @oct: Octeon device private data structure.
389 * @oq: Octeon Rx queue data structure.
390 * @pkts_to_process: number of packets to be processed.
391 *
392 * Process the new packets in Rx queue.
393 * Packets larger than single Rx buffer arrive in consecutive descriptors.
394 * But, count returned by the API only accounts full packets, not fragments.
395 *
396 * Return: number of packets processed and pushed to stack.
397 */
398static int __octep_oq_process_rx(struct octep_device *oct,
399 struct octep_oq *oq, u16 pkts_to_process)
400{
401 struct octep_oq_resp_hw_ext *resp_hw_ext = NULL;
402 netdev_features_t feat = oq->netdev->features;
403 struct octep_rx_buffer *buff_info;
404 struct octep_oq_resp_hw *resp_hw;
405 u32 pkt, rx_bytes, desc_used;
406 struct sk_buff *skb;
407 u16 data_offset;
408 u16 rx_ol_flags;
409 u32 read_idx;
410
411 read_idx = oq->host_read_idx;
412 rx_bytes = 0;
413 desc_used = 0;
414 for (pkt = 0; pkt < pkts_to_process; pkt++) {
415 buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];
416 resp_hw = page_address(buff_info->page);
417
418 /* Swap the length field that is in Big-Endian to CPU */
419 buff_info->len = be64_to_cpu(resp_hw->length);
420 if (oct->conf->fw_info.rx_ol_flags) {
421 /* Extended response header is immediately after
422 * response header (resp_hw)
423 */
424 resp_hw_ext = (struct octep_oq_resp_hw_ext *)
425 (resp_hw + 1);
426 buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE;
427 /* Packet Data is immediately after
428 * extended response header.
429 */
430 data_offset = OCTEP_OQ_RESP_HW_SIZE +
431 OCTEP_OQ_RESP_HW_EXT_SIZE;
432 rx_ol_flags = resp_hw_ext->rx_ol_flags;
433 } else {
434 /* Data is immediately after
435 * Hardware Rx response header.
436 */
437 data_offset = OCTEP_OQ_RESP_HW_SIZE;
438 rx_ol_flags = 0;
439 }
440
441 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);
442
443 skb = build_skb((void *)resp_hw, PAGE_SIZE);
444 if (!skb) {
445 octep_oq_drop_rx(oq, buff_info,
446 &read_idx, &desc_used);
447 oq->stats->alloc_failures++;
448 continue;
449 }
450 skb_reserve(skb, data_offset);
451
452 rx_bytes += buff_info->len;
453
454 if (buff_info->len <= oq->max_single_buffer_size) {
455 skb_put(skb, buff_info->len);
456 } else {
457 struct skb_shared_info *shinfo;
458 u16 data_len;
459
460 /* Head fragment includes response header(s);
461 * subsequent fragments contains only data.
462 */
463 skb_put(skb, oq->max_single_buffer_size);
464 shinfo = skb_shinfo(skb);
465 data_len = buff_info->len - oq->max_single_buffer_size;
466 while (data_len) {
467 buff_info = (struct octep_rx_buffer *)
468 &oq->buff_info[read_idx];
469 if (data_len < oq->buffer_size) {
470 buff_info->len = data_len;
471 data_len = 0;
472 } else {
473 buff_info->len = oq->buffer_size;
474 data_len -= oq->buffer_size;
475 }
476
477 skb_add_rx_frag(skb, shinfo->nr_frags,
478 buff_info->page, 0,
479 buff_info->len,
480 buff_info->len);
481
482 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);
483 }
484 }
485
486 skb->dev = oq->netdev;
487 skb->protocol = eth_type_trans(skb, skb->dev);
488 if (feat & NETIF_F_RXCSUM &&
489 OCTEP_RX_CSUM_VERIFIED(rx_ol_flags))
490 skb->ip_summed = CHECKSUM_UNNECESSARY;
491 else
492 skb->ip_summed = CHECKSUM_NONE;
493 napi_gro_receive(oq->napi, skb);
494 }
495
496 oq->host_read_idx = read_idx;
497 oq->refill_count += desc_used;
498 oq->stats->packets += pkt;
499 oq->stats->bytes += rx_bytes;
500
501 return pkt;
502}
503
504/**
505 * octep_oq_process_rx() - Process Rx queue.
506 *
507 * @oq: Octeon Rx queue data structure.
508 * @budget: max number of packets can be processed in one invocation.
509 *
510 * Check for newly received packets and process them.
511 * Keeps checking for new packets until budget is used or no new packets seen.
512 *
513 * Return: number of packets processed.
514 */
515int octep_oq_process_rx(struct octep_oq *oq, int budget)
516{
517 u32 pkts_available, pkts_processed, total_pkts_processed;
518 struct octep_device *oct = oq->octep_dev;
519
520 pkts_available = 0;
521 pkts_processed = 0;
522 total_pkts_processed = 0;
523 while (total_pkts_processed < budget) {
524 /* update pending count only when current one exhausted */
525 if (oq->pkts_pending == 0)
526 octep_oq_check_hw_for_pkts(oct, oq);
527 pkts_available = min(budget - total_pkts_processed,
528 oq->pkts_pending);
529 if (!pkts_available)
530 break;
531
532 pkts_processed = __octep_oq_process_rx(oct, oq,
533 pkts_available);
534 oq->pkts_pending -= pkts_processed;
535 total_pkts_processed += pkts_processed;
536 }
537
538 if (oq->refill_count >= oq->refill_threshold) {
539 u32 desc_refilled = octep_oq_refill(oct, oq);
540
541 /* flush pending writes before updating credits */
542 wmb();
543 writel(desc_refilled, oq->pkts_credit_reg);
544 }
545
546 return total_pkts_processed;
547}