Loading...
1/***********************license start***************
2 * Author: Cavium Networks
3 *
4 * Contact: support@caviumnetworks.com
5 * This file is part of the OCTEON SDK
6 *
7 * Copyright (c) 2003-2008 Cavium Networks
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
17 * details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this file; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
22 * or visit http://www.gnu.org/licenses/.
23 *
24 * This file may also be available under a different license from Cavium.
25 * Contact Cavium Networks for more information
26 ***********************license end**************************************/
27
28/*
29 *
30 * Support functions for managing command queues used for
31 * various hardware blocks.
32 *
33 * The common command queue infrastructure abstracts out the
34 * software necessary for adding to Octeon's chained queue
35 * structures. These structures are used for commands to the
36 * PKO, ZIP, DFA, RAID, and DMA engine blocks. Although each
37 * hardware unit takes commands and CSRs of different types,
38 * they all use basic linked command buffers to store the
39 * pending request. In general, users of the CVMX API don't
40 * call cvmx-cmd-queue functions directly. Instead the hardware
41 * unit specific wrapper should be used. The wrappers perform
42 * unit specific validation and CSR writes to submit the
43 * commands.
44 *
45 * Even though most software will never directly interact with
46 * cvmx-cmd-queue, knowledge of its internal working can help
47 * in diagnosing performance problems and help with debugging.
48 *
49 * Command queue pointers are stored in a global named block
50 * called "cvmx_cmd_queues". Except for the PKO queues, each
51 * hardware queue is stored in its own cache line to reduce SMP
52 * contention on spin locks. The PKO queues are stored such that
53 * every 16th queue is next to each other in memory. This scheme
54 * allows for queues being in separate cache lines when there
55 * are low number of queues per port. With 16 queues per port,
56 * the first queue for each port is in the same cache area. The
57 * second queues for each port are in another area, etc. This
58 * allows software to implement very efficient lockless PKO with
59 * 16 queues per port using a minimum of cache lines per core.
60 * All queues for a given core will be isolated in the same
61 * cache area.
62 *
63 * In addition to the memory pointer layout, cvmx-cmd-queue
64 * provides an optimized fair ll/sc locking mechanism for the
65 * queues. The lock uses a "ticket / now serving" model to
66 * maintain fair order on contended locks. In addition, it uses
67 * predicted locking time to limit cache contention. When a core
68 * know it must wait in line for a lock, it spins on the
69 * internal cycle counter to completely eliminate any causes of
70 * bus traffic.
71 *
72 */
73
74#ifndef __CVMX_CMD_QUEUE_H__
75#define __CVMX_CMD_QUEUE_H__
76
77#include <linux/prefetch.h>
78
79#include "cvmx-fpa.h"
80/**
81 * By default we disable the max depth support. Most programs
82 * don't use it and it slows down the command queue processing
83 * significantly.
84 */
85#ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH
86#define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0
87#endif
88
89/**
90 * Enumeration representing all hardware blocks that use command
91 * queues. Each hardware block has up to 65536 sub identifiers for
92 * multiple command queues. Not all chips support all hardware
93 * units.
94 */
95typedef enum {
96 CVMX_CMD_QUEUE_PKO_BASE = 0x00000,
97
98#define CVMX_CMD_QUEUE_PKO(queue) \
99 ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff&(queue))))
100
101 CVMX_CMD_QUEUE_ZIP = 0x10000,
102 CVMX_CMD_QUEUE_DFA = 0x20000,
103 CVMX_CMD_QUEUE_RAID = 0x30000,
104 CVMX_CMD_QUEUE_DMA_BASE = 0x40000,
105
106#define CVMX_CMD_QUEUE_DMA(queue) \
107 ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff&(queue))))
108
109 CVMX_CMD_QUEUE_END = 0x50000,
110} cvmx_cmd_queue_id_t;
111
112/**
113 * Command write operations can fail if the command queue needs
114 * a new buffer and the associated FPA pool is empty. It can also
115 * fail if the number of queued command words reaches the maximum
116 * set at initialization.
117 */
118typedef enum {
119 CVMX_CMD_QUEUE_SUCCESS = 0,
120 CVMX_CMD_QUEUE_NO_MEMORY = -1,
121 CVMX_CMD_QUEUE_FULL = -2,
122 CVMX_CMD_QUEUE_INVALID_PARAM = -3,
123 CVMX_CMD_QUEUE_ALREADY_SETUP = -4,
124} cvmx_cmd_queue_result_t;
125
126typedef struct {
127 /* You have lock when this is your ticket */
128 uint8_t now_serving;
129 uint64_t unused1:24;
130 /* Maximum outstanding command words */
131 uint32_t max_depth;
132 /* FPA pool buffers come from */
133 uint64_t fpa_pool:3;
134 /* Top of command buffer pointer shifted 7 */
135 uint64_t base_ptr_div128:29;
136 uint64_t unused2:6;
137 /* FPA buffer size in 64bit words minus 1 */
138 uint64_t pool_size_m1:13;
139 /* Number of commands already used in buffer */
140 uint64_t index:13;
141} __cvmx_cmd_queue_state_t;
142
143/**
144 * This structure contains the global state of all command queues.
145 * It is stored in a bootmem named block and shared by all
146 * applications running on Octeon. Tickets are stored in a differnet
147 * cahce line that queue information to reduce the contention on the
148 * ll/sc used to get a ticket. If this is not the case, the update
149 * of queue state causes the ll/sc to fail quite often.
150 */
151typedef struct {
152 uint64_t ticket[(CVMX_CMD_QUEUE_END >> 16) * 256];
153 __cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END >> 16) * 256];
154} __cvmx_cmd_queue_all_state_t;
155
156/**
157 * Initialize a command queue for use. The initial FPA buffer is
158 * allocated and the hardware unit is configured to point to the
159 * new command queue.
160 *
161 * @queue_id: Hardware command queue to initialize.
162 * @max_depth: Maximum outstanding commands that can be queued.
163 * @fpa_pool: FPA pool the command queues should come from.
164 * @pool_size: Size of each buffer in the FPA pool (bytes)
165 *
166 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
167 */
168cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id,
169 int max_depth, int fpa_pool,
170 int pool_size);
171
172/**
173 * Shutdown a queue a free it's command buffers to the FPA. The
174 * hardware connected to the queue must be stopped before this
175 * function is called.
176 *
177 * @queue_id: Queue to shutdown
178 *
179 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
180 */
181cvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t queue_id);
182
183/**
184 * Return the number of command words pending in the queue. This
185 * function may be relatively slow for some hardware units.
186 *
187 * @queue_id: Hardware command queue to query
188 *
189 * Returns Number of outstanding commands
190 */
191int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id);
192
193/**
194 * Return the command buffer to be written to. The purpose of this
195 * function is to allow CVMX routine access t othe low level buffer
196 * for initial hardware setup. User applications should not call this
197 * function directly.
198 *
199 * @queue_id: Command queue to query
200 *
201 * Returns Command buffer or NULL on failure
202 */
203void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id);
204
205/**
206 * Get the index into the state arrays for the supplied queue id.
207 *
208 * @queue_id: Queue ID to get an index for
209 *
210 * Returns Index into the state arrays
211 */
212static inline int __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id)
213{
214 /*
215 * Warning: This code currently only works with devices that
216 * have 256 queues or less. Devices with more than 16 queues
217 * are laid out in memory to allow cores quick access to
218 * every 16th queue. This reduces cache thrashing when you are
219 * running 16 queues per port to support lockless operation.
220 */
221 int unit = queue_id >> 16;
222 int q = (queue_id >> 4) & 0xf;
223 int core = queue_id & 0xf;
224 return unit * 256 + core * 16 + q;
225}
226
227/**
228 * Lock the supplied queue so nobody else is updating it at the same
229 * time as us.
230 *
231 * @queue_id: Queue ID to lock
232 * @qptr: Pointer to the queue's global state
233 */
234static inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id,
235 __cvmx_cmd_queue_state_t *qptr)
236{
237 extern __cvmx_cmd_queue_all_state_t
238 *__cvmx_cmd_queue_state_ptr;
239 int tmp;
240 int my_ticket;
241 prefetch(qptr);
242 asm volatile (
243 ".set push\n"
244 ".set noreorder\n"
245 "1:\n"
246 /* Atomic add one to ticket_ptr */
247 "ll %[my_ticket], %[ticket_ptr]\n"
248 /* and store the original value */
249 "li %[ticket], 1\n"
250 /* in my_ticket */
251 "baddu %[ticket], %[my_ticket]\n"
252 "sc %[ticket], %[ticket_ptr]\n"
253 "beqz %[ticket], 1b\n"
254 " nop\n"
255 /* Load the current now_serving ticket */
256 "lbu %[ticket], %[now_serving]\n"
257 "2:\n"
258 /* Jump out if now_serving == my_ticket */
259 "beq %[ticket], %[my_ticket], 4f\n"
260 /* Find out how many tickets are in front of me */
261 " subu %[ticket], %[my_ticket], %[ticket]\n"
262 /* Use tickets in front of me minus one to delay */
263 "subu %[ticket], 1\n"
264 /* Delay will be ((tickets in front)-1)*32 loops */
265 "cins %[ticket], %[ticket], 5, 7\n"
266 "3:\n"
267 /* Loop here until our ticket might be up */
268 "bnez %[ticket], 3b\n"
269 " subu %[ticket], 1\n"
270 /* Jump back up to check out ticket again */
271 "b 2b\n"
272 /* Load the current now_serving ticket */
273 " lbu %[ticket], %[now_serving]\n"
274 "4:\n"
275 ".set pop\n" :
276 [ticket_ptr] "=m"(__cvmx_cmd_queue_state_ptr->ticket[__cvmx_cmd_queue_get_index(queue_id)]),
277 [now_serving] "=m"(qptr->now_serving), [ticket] "=r"(tmp),
278 [my_ticket] "=r"(my_ticket)
279 );
280}
281
282/**
283 * Unlock the queue, flushing all writes.
284 *
285 * @qptr: Queue to unlock
286 */
287static inline void __cvmx_cmd_queue_unlock(__cvmx_cmd_queue_state_t *qptr)
288{
289 qptr->now_serving++;
290 CVMX_SYNCWS;
291}
292
293/**
294 * Get the queue state structure for the given queue id
295 *
296 * @queue_id: Queue id to get
297 *
298 * Returns Queue structure or NULL on failure
299 */
300static inline __cvmx_cmd_queue_state_t
301 *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id)
302{
303 extern __cvmx_cmd_queue_all_state_t
304 *__cvmx_cmd_queue_state_ptr;
305 return &__cvmx_cmd_queue_state_ptr->
306 state[__cvmx_cmd_queue_get_index(queue_id)];
307}
308
309/**
310 * Write an arbitrary number of command words to a command queue.
311 * This is a generic function; the fixed number of command word
312 * functions yield higher performance.
313 *
314 * @queue_id: Hardware command queue to write to
315 * @use_locking:
316 * Use internal locking to ensure exclusive access for queue
317 * updates. If you don't use this locking you must ensure
318 * exclusivity some other way. Locking is strongly recommended.
319 * @cmd_count: Number of command words to write
320 * @cmds: Array of commands to write
321 *
322 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
323 */
324static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write(cvmx_cmd_queue_id_t
325 queue_id,
326 int use_locking,
327 int cmd_count,
328 uint64_t *cmds)
329{
330 __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
331
332 /* Make sure nobody else is updating the same queue */
333 if (likely(use_locking))
334 __cvmx_cmd_queue_lock(queue_id, qptr);
335
336 /*
337 * If a max queue length was specified then make sure we don't
338 * exceed it. If any part of the command would be below the
339 * limit we allow it.
340 */
341 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
342 if (unlikely
343 (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
344 if (likely(use_locking))
345 __cvmx_cmd_queue_unlock(qptr);
346 return CVMX_CMD_QUEUE_FULL;
347 }
348 }
349
350 /*
351 * Normally there is plenty of room in the current buffer for
352 * the command.
353 */
354 if (likely(qptr->index + cmd_count < qptr->pool_size_m1)) {
355 uint64_t *ptr =
356 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
357 base_ptr_div128 << 7);
358 ptr += qptr->index;
359 qptr->index += cmd_count;
360 while (cmd_count--)
361 *ptr++ = *cmds++;
362 } else {
363 uint64_t *ptr;
364 int count;
365 /*
366 * We need a new command buffer. Fail if there isn't
367 * one available.
368 */
369 uint64_t *new_buffer =
370 (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
371 if (unlikely(new_buffer == NULL)) {
372 if (likely(use_locking))
373 __cvmx_cmd_queue_unlock(qptr);
374 return CVMX_CMD_QUEUE_NO_MEMORY;
375 }
376 ptr =
377 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
378 base_ptr_div128 << 7);
379 /*
380 * Figure out how many command words will fit in this
381 * buffer. One location will be needed for the next
382 * buffer pointer.
383 */
384 count = qptr->pool_size_m1 - qptr->index;
385 ptr += qptr->index;
386 cmd_count -= count;
387 while (count--)
388 *ptr++ = *cmds++;
389 *ptr = cvmx_ptr_to_phys(new_buffer);
390 /*
391 * The current buffer is full and has a link to the
392 * next buffer. Time to write the rest of the commands
393 * into the new buffer.
394 */
395 qptr->base_ptr_div128 = *ptr >> 7;
396 qptr->index = cmd_count;
397 ptr = new_buffer;
398 while (cmd_count--)
399 *ptr++ = *cmds++;
400 }
401
402 /* All updates are complete. Release the lock and return */
403 if (likely(use_locking))
404 __cvmx_cmd_queue_unlock(qptr);
405 return CVMX_CMD_QUEUE_SUCCESS;
406}
407
408/**
409 * Simple function to write two command words to a command
410 * queue.
411 *
412 * @queue_id: Hardware command queue to write to
413 * @use_locking:
414 * Use internal locking to ensure exclusive access for queue
415 * updates. If you don't use this locking you must ensure
416 * exclusivity some other way. Locking is strongly recommended.
417 * @cmd1: Command
418 * @cmd2: Command
419 *
420 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
421 */
422static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t
423 queue_id,
424 int use_locking,
425 uint64_t cmd1,
426 uint64_t cmd2)
427{
428 __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
429
430 /* Make sure nobody else is updating the same queue */
431 if (likely(use_locking))
432 __cvmx_cmd_queue_lock(queue_id, qptr);
433
434 /*
435 * If a max queue length was specified then make sure we don't
436 * exceed it. If any part of the command would be below the
437 * limit we allow it.
438 */
439 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
440 if (unlikely
441 (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
442 if (likely(use_locking))
443 __cvmx_cmd_queue_unlock(qptr);
444 return CVMX_CMD_QUEUE_FULL;
445 }
446 }
447
448 /*
449 * Normally there is plenty of room in the current buffer for
450 * the command.
451 */
452 if (likely(qptr->index + 2 < qptr->pool_size_m1)) {
453 uint64_t *ptr =
454 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
455 base_ptr_div128 << 7);
456 ptr += qptr->index;
457 qptr->index += 2;
458 ptr[0] = cmd1;
459 ptr[1] = cmd2;
460 } else {
461 uint64_t *ptr;
462 /*
463 * Figure out how many command words will fit in this
464 * buffer. One location will be needed for the next
465 * buffer pointer.
466 */
467 int count = qptr->pool_size_m1 - qptr->index;
468 /*
469 * We need a new command buffer. Fail if there isn't
470 * one available.
471 */
472 uint64_t *new_buffer =
473 (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
474 if (unlikely(new_buffer == NULL)) {
475 if (likely(use_locking))
476 __cvmx_cmd_queue_unlock(qptr);
477 return CVMX_CMD_QUEUE_NO_MEMORY;
478 }
479 count--;
480 ptr =
481 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
482 base_ptr_div128 << 7);
483 ptr += qptr->index;
484 *ptr++ = cmd1;
485 if (likely(count))
486 *ptr++ = cmd2;
487 *ptr = cvmx_ptr_to_phys(new_buffer);
488 /*
489 * The current buffer is full and has a link to the
490 * next buffer. Time to write the rest of the commands
491 * into the new buffer.
492 */
493 qptr->base_ptr_div128 = *ptr >> 7;
494 qptr->index = 0;
495 if (unlikely(count == 0)) {
496 qptr->index = 1;
497 new_buffer[0] = cmd2;
498 }
499 }
500
501 /* All updates are complete. Release the lock and return */
502 if (likely(use_locking))
503 __cvmx_cmd_queue_unlock(qptr);
504 return CVMX_CMD_QUEUE_SUCCESS;
505}
506
507/**
508 * Simple function to write three command words to a command
509 * queue.
510 *
511 * @queue_id: Hardware command queue to write to
512 * @use_locking:
513 * Use internal locking to ensure exclusive access for queue
514 * updates. If you don't use this locking you must ensure
515 * exclusivity some other way. Locking is strongly recommended.
516 * @cmd1: Command
517 * @cmd2: Command
518 * @cmd3: Command
519 *
520 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
521 */
522static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t
523 queue_id,
524 int use_locking,
525 uint64_t cmd1,
526 uint64_t cmd2,
527 uint64_t cmd3)
528{
529 __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
530
531 /* Make sure nobody else is updating the same queue */
532 if (likely(use_locking))
533 __cvmx_cmd_queue_lock(queue_id, qptr);
534
535 /*
536 * If a max queue length was specified then make sure we don't
537 * exceed it. If any part of the command would be below the
538 * limit we allow it.
539 */
540 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
541 if (unlikely
542 (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
543 if (likely(use_locking))
544 __cvmx_cmd_queue_unlock(qptr);
545 return CVMX_CMD_QUEUE_FULL;
546 }
547 }
548
549 /*
550 * Normally there is plenty of room in the current buffer for
551 * the command.
552 */
553 if (likely(qptr->index + 3 < qptr->pool_size_m1)) {
554 uint64_t *ptr =
555 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
556 base_ptr_div128 << 7);
557 ptr += qptr->index;
558 qptr->index += 3;
559 ptr[0] = cmd1;
560 ptr[1] = cmd2;
561 ptr[2] = cmd3;
562 } else {
563 uint64_t *ptr;
564 /*
565 * Figure out how many command words will fit in this
566 * buffer. One location will be needed for the next
567 * buffer pointer
568 */
569 int count = qptr->pool_size_m1 - qptr->index;
570 /*
571 * We need a new command buffer. Fail if there isn't
572 * one available
573 */
574 uint64_t *new_buffer =
575 (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
576 if (unlikely(new_buffer == NULL)) {
577 if (likely(use_locking))
578 __cvmx_cmd_queue_unlock(qptr);
579 return CVMX_CMD_QUEUE_NO_MEMORY;
580 }
581 count--;
582 ptr =
583 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
584 base_ptr_div128 << 7);
585 ptr += qptr->index;
586 *ptr++ = cmd1;
587 if (count) {
588 *ptr++ = cmd2;
589 if (count > 1)
590 *ptr++ = cmd3;
591 }
592 *ptr = cvmx_ptr_to_phys(new_buffer);
593 /*
594 * The current buffer is full and has a link to the
595 * next buffer. Time to write the rest of the commands
596 * into the new buffer.
597 */
598 qptr->base_ptr_div128 = *ptr >> 7;
599 qptr->index = 0;
600 ptr = new_buffer;
601 if (count == 0) {
602 *ptr++ = cmd2;
603 qptr->index++;
604 }
605 if (count < 2) {
606 *ptr++ = cmd3;
607 qptr->index++;
608 }
609 }
610
611 /* All updates are complete. Release the lock and return */
612 if (likely(use_locking))
613 __cvmx_cmd_queue_unlock(qptr);
614 return CVMX_CMD_QUEUE_SUCCESS;
615}
616
617#endif /* __CVMX_CMD_QUEUE_H__ */
1/***********************license start***************
2 * Author: Cavium Networks
3 *
4 * Contact: support@caviumnetworks.com
5 * This file is part of the OCTEON SDK
6 *
7 * Copyright (c) 2003-2008 Cavium Networks
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
17 * details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this file; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
22 * or visit http://www.gnu.org/licenses/.
23 *
24 * This file may also be available under a different license from Cavium.
25 * Contact Cavium Networks for more information
26 ***********************license end**************************************/
27
28/*
29 *
30 * Support functions for managing command queues used for
31 * various hardware blocks.
32 *
33 * The common command queue infrastructure abstracts out the
34 * software necessary for adding to Octeon's chained queue
35 * structures. These structures are used for commands to the
36 * PKO, ZIP, DFA, RAID, and DMA engine blocks. Although each
37 * hardware unit takes commands and CSRs of different types,
38 * they all use basic linked command buffers to store the
39 * pending request. In general, users of the CVMX API don't
40 * call cvmx-cmd-queue functions directly. Instead the hardware
41 * unit specific wrapper should be used. The wrappers perform
42 * unit specific validation and CSR writes to submit the
43 * commands.
44 *
45 * Even though most software will never directly interact with
46 * cvmx-cmd-queue, knowledge of its internal working can help
47 * in diagnosing performance problems and help with debugging.
48 *
49 * Command queue pointers are stored in a global named block
50 * called "cvmx_cmd_queues". Except for the PKO queues, each
51 * hardware queue is stored in its own cache line to reduce SMP
52 * contention on spin locks. The PKO queues are stored such that
53 * every 16th queue is next to each other in memory. This scheme
54 * allows for queues being in separate cache lines when there
55 * are low number of queues per port. With 16 queues per port,
56 * the first queue for each port is in the same cache area. The
57 * second queues for each port are in another area, etc. This
58 * allows software to implement very efficient lockless PKO with
59 * 16 queues per port using a minimum of cache lines per core.
60 * All queues for a given core will be isolated in the same
61 * cache area.
62 *
63 * In addition to the memory pointer layout, cvmx-cmd-queue
64 * provides an optimized fair ll/sc locking mechanism for the
65 * queues. The lock uses a "ticket / now serving" model to
66 * maintain fair order on contended locks. In addition, it uses
67 * predicted locking time to limit cache contention. When a core
68 * know it must wait in line for a lock, it spins on the
69 * internal cycle counter to completely eliminate any causes of
70 * bus traffic.
71 *
72 */
73
74#ifndef __CVMX_CMD_QUEUE_H__
75#define __CVMX_CMD_QUEUE_H__
76
77#include <linux/prefetch.h>
78
79#include <asm/compiler.h>
80
81#include <asm/octeon/cvmx-fpa.h>
82/**
83 * By default we disable the max depth support. Most programs
84 * don't use it and it slows down the command queue processing
85 * significantly.
86 */
87#ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH
88#define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0
89#endif
90
91/**
92 * Enumeration representing all hardware blocks that use command
93 * queues. Each hardware block has up to 65536 sub identifiers for
94 * multiple command queues. Not all chips support all hardware
95 * units.
96 */
97typedef enum {
98 CVMX_CMD_QUEUE_PKO_BASE = 0x00000,
99
100#define CVMX_CMD_QUEUE_PKO(queue) \
101 ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff&(queue))))
102
103 CVMX_CMD_QUEUE_ZIP = 0x10000,
104 CVMX_CMD_QUEUE_DFA = 0x20000,
105 CVMX_CMD_QUEUE_RAID = 0x30000,
106 CVMX_CMD_QUEUE_DMA_BASE = 0x40000,
107
108#define CVMX_CMD_QUEUE_DMA(queue) \
109 ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff&(queue))))
110
111 CVMX_CMD_QUEUE_END = 0x50000,
112} cvmx_cmd_queue_id_t;
113
114/**
115 * Command write operations can fail if the command queue needs
116 * a new buffer and the associated FPA pool is empty. It can also
117 * fail if the number of queued command words reaches the maximum
118 * set at initialization.
119 */
120typedef enum {
121 CVMX_CMD_QUEUE_SUCCESS = 0,
122 CVMX_CMD_QUEUE_NO_MEMORY = -1,
123 CVMX_CMD_QUEUE_FULL = -2,
124 CVMX_CMD_QUEUE_INVALID_PARAM = -3,
125 CVMX_CMD_QUEUE_ALREADY_SETUP = -4,
126} cvmx_cmd_queue_result_t;
127
128typedef struct {
129 /* You have lock when this is your ticket */
130 uint8_t now_serving;
131 uint64_t unused1:24;
132 /* Maximum outstanding command words */
133 uint32_t max_depth;
134 /* FPA pool buffers come from */
135 uint64_t fpa_pool:3;
136 /* Top of command buffer pointer shifted 7 */
137 uint64_t base_ptr_div128:29;
138 uint64_t unused2:6;
139 /* FPA buffer size in 64bit words minus 1 */
140 uint64_t pool_size_m1:13;
141 /* Number of commands already used in buffer */
142 uint64_t index:13;
143} __cvmx_cmd_queue_state_t;
144
145/**
146 * This structure contains the global state of all command queues.
147 * It is stored in a bootmem named block and shared by all
148 * applications running on Octeon. Tickets are stored in a differnet
149 * cahce line that queue information to reduce the contention on the
150 * ll/sc used to get a ticket. If this is not the case, the update
151 * of queue state causes the ll/sc to fail quite often.
152 */
153typedef struct {
154 uint64_t ticket[(CVMX_CMD_QUEUE_END >> 16) * 256];
155 __cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END >> 16) * 256];
156} __cvmx_cmd_queue_all_state_t;
157
158/**
159 * Initialize a command queue for use. The initial FPA buffer is
160 * allocated and the hardware unit is configured to point to the
161 * new command queue.
162 *
163 * @queue_id: Hardware command queue to initialize.
164 * @max_depth: Maximum outstanding commands that can be queued.
165 * @fpa_pool: FPA pool the command queues should come from.
166 * @pool_size: Size of each buffer in the FPA pool (bytes)
167 *
168 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
169 */
170cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id,
171 int max_depth, int fpa_pool,
172 int pool_size);
173
174/**
175 * Shutdown a queue a free it's command buffers to the FPA. The
176 * hardware connected to the queue must be stopped before this
177 * function is called.
178 *
179 * @queue_id: Queue to shutdown
180 *
181 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
182 */
183cvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t queue_id);
184
185/**
186 * Return the number of command words pending in the queue. This
187 * function may be relatively slow for some hardware units.
188 *
189 * @queue_id: Hardware command queue to query
190 *
191 * Returns Number of outstanding commands
192 */
193int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id);
194
195/**
196 * Return the command buffer to be written to. The purpose of this
197 * function is to allow CVMX routine access t othe low level buffer
198 * for initial hardware setup. User applications should not call this
199 * function directly.
200 *
201 * @queue_id: Command queue to query
202 *
203 * Returns Command buffer or NULL on failure
204 */
205void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id);
206
207/**
208 * Get the index into the state arrays for the supplied queue id.
209 *
210 * @queue_id: Queue ID to get an index for
211 *
212 * Returns Index into the state arrays
213 */
214static inline int __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id)
215{
216 /*
217 * Warning: This code currently only works with devices that
218 * have 256 queues or less. Devices with more than 16 queues
219 * are laid out in memory to allow cores quick access to
220 * every 16th queue. This reduces cache thrashing when you are
221 * running 16 queues per port to support lockless operation.
222 */
223 int unit = queue_id >> 16;
224 int q = (queue_id >> 4) & 0xf;
225 int core = queue_id & 0xf;
226 return unit * 256 + core * 16 + q;
227}
228
229/**
230 * Lock the supplied queue so nobody else is updating it at the same
231 * time as us.
232 *
233 * @queue_id: Queue ID to lock
234 * @qptr: Pointer to the queue's global state
235 */
236static inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id,
237 __cvmx_cmd_queue_state_t *qptr)
238{
239 extern __cvmx_cmd_queue_all_state_t
240 *__cvmx_cmd_queue_state_ptr;
241 int tmp;
242 int my_ticket;
243 prefetch(qptr);
244 asm volatile (
245 ".set push\n"
246 ".set noreorder\n"
247 "1:\n"
248 /* Atomic add one to ticket_ptr */
249 "ll %[my_ticket], %[ticket_ptr]\n"
250 /* and store the original value */
251 "li %[ticket], 1\n"
252 /* in my_ticket */
253 "baddu %[ticket], %[my_ticket]\n"
254 "sc %[ticket], %[ticket_ptr]\n"
255 "beqz %[ticket], 1b\n"
256 " nop\n"
257 /* Load the current now_serving ticket */
258 "lbu %[ticket], %[now_serving]\n"
259 "2:\n"
260 /* Jump out if now_serving == my_ticket */
261 "beq %[ticket], %[my_ticket], 4f\n"
262 /* Find out how many tickets are in front of me */
263 " subu %[ticket], %[my_ticket], %[ticket]\n"
264 /* Use tickets in front of me minus one to delay */
265 "subu %[ticket], 1\n"
266 /* Delay will be ((tickets in front)-1)*32 loops */
267 "cins %[ticket], %[ticket], 5, 7\n"
268 "3:\n"
269 /* Loop here until our ticket might be up */
270 "bnez %[ticket], 3b\n"
271 " subu %[ticket], 1\n"
272 /* Jump back up to check out ticket again */
273 "b 2b\n"
274 /* Load the current now_serving ticket */
275 " lbu %[ticket], %[now_serving]\n"
276 "4:\n"
277 ".set pop\n" :
278 [ticket_ptr] "=" GCC_OFF_SMALL_ASM()(__cvmx_cmd_queue_state_ptr->ticket[__cvmx_cmd_queue_get_index(queue_id)]),
279 [now_serving] "=m"(qptr->now_serving), [ticket] "=r"(tmp),
280 [my_ticket] "=r"(my_ticket)
281 );
282}
283
284/**
285 * Unlock the queue, flushing all writes.
286 *
287 * @qptr: Queue to unlock
288 */
289static inline void __cvmx_cmd_queue_unlock(__cvmx_cmd_queue_state_t *qptr)
290{
291 qptr->now_serving++;
292 CVMX_SYNCWS;
293}
294
295/**
296 * Get the queue state structure for the given queue id
297 *
298 * @queue_id: Queue id to get
299 *
300 * Returns Queue structure or NULL on failure
301 */
302static inline __cvmx_cmd_queue_state_t
303 *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id)
304{
305 extern __cvmx_cmd_queue_all_state_t
306 *__cvmx_cmd_queue_state_ptr;
307 return &__cvmx_cmd_queue_state_ptr->
308 state[__cvmx_cmd_queue_get_index(queue_id)];
309}
310
311/**
312 * Write an arbitrary number of command words to a command queue.
313 * This is a generic function; the fixed number of command word
314 * functions yield higher performance.
315 *
316 * @queue_id: Hardware command queue to write to
317 * @use_locking:
318 * Use internal locking to ensure exclusive access for queue
319 * updates. If you don't use this locking you must ensure
320 * exclusivity some other way. Locking is strongly recommended.
321 * @cmd_count: Number of command words to write
322 * @cmds: Array of commands to write
323 *
324 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
325 */
326static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write(cvmx_cmd_queue_id_t
327 queue_id,
328 int use_locking,
329 int cmd_count,
330 uint64_t *cmds)
331{
332 __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
333
334 /* Make sure nobody else is updating the same queue */
335 if (likely(use_locking))
336 __cvmx_cmd_queue_lock(queue_id, qptr);
337
338 /*
339 * If a max queue length was specified then make sure we don't
340 * exceed it. If any part of the command would be below the
341 * limit we allow it.
342 */
343 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
344 if (unlikely
345 (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
346 if (likely(use_locking))
347 __cvmx_cmd_queue_unlock(qptr);
348 return CVMX_CMD_QUEUE_FULL;
349 }
350 }
351
352 /*
353 * Normally there is plenty of room in the current buffer for
354 * the command.
355 */
356 if (likely(qptr->index + cmd_count < qptr->pool_size_m1)) {
357 uint64_t *ptr =
358 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
359 base_ptr_div128 << 7);
360 ptr += qptr->index;
361 qptr->index += cmd_count;
362 while (cmd_count--)
363 *ptr++ = *cmds++;
364 } else {
365 uint64_t *ptr;
366 int count;
367 /*
368 * We need a new command buffer. Fail if there isn't
369 * one available.
370 */
371 uint64_t *new_buffer =
372 (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
373 if (unlikely(new_buffer == NULL)) {
374 if (likely(use_locking))
375 __cvmx_cmd_queue_unlock(qptr);
376 return CVMX_CMD_QUEUE_NO_MEMORY;
377 }
378 ptr =
379 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
380 base_ptr_div128 << 7);
381 /*
382 * Figure out how many command words will fit in this
383 * buffer. One location will be needed for the next
384 * buffer pointer.
385 */
386 count = qptr->pool_size_m1 - qptr->index;
387 ptr += qptr->index;
388 cmd_count -= count;
389 while (count--)
390 *ptr++ = *cmds++;
391 *ptr = cvmx_ptr_to_phys(new_buffer);
392 /*
393 * The current buffer is full and has a link to the
394 * next buffer. Time to write the rest of the commands
395 * into the new buffer.
396 */
397 qptr->base_ptr_div128 = *ptr >> 7;
398 qptr->index = cmd_count;
399 ptr = new_buffer;
400 while (cmd_count--)
401 *ptr++ = *cmds++;
402 }
403
404 /* All updates are complete. Release the lock and return */
405 if (likely(use_locking))
406 __cvmx_cmd_queue_unlock(qptr);
407 return CVMX_CMD_QUEUE_SUCCESS;
408}
409
410/**
411 * Simple function to write two command words to a command
412 * queue.
413 *
414 * @queue_id: Hardware command queue to write to
415 * @use_locking:
416 * Use internal locking to ensure exclusive access for queue
417 * updates. If you don't use this locking you must ensure
418 * exclusivity some other way. Locking is strongly recommended.
419 * @cmd1: Command
420 * @cmd2: Command
421 *
422 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
423 */
424static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t
425 queue_id,
426 int use_locking,
427 uint64_t cmd1,
428 uint64_t cmd2)
429{
430 __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
431
432 /* Make sure nobody else is updating the same queue */
433 if (likely(use_locking))
434 __cvmx_cmd_queue_lock(queue_id, qptr);
435
436 /*
437 * If a max queue length was specified then make sure we don't
438 * exceed it. If any part of the command would be below the
439 * limit we allow it.
440 */
441 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
442 if (unlikely
443 (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
444 if (likely(use_locking))
445 __cvmx_cmd_queue_unlock(qptr);
446 return CVMX_CMD_QUEUE_FULL;
447 }
448 }
449
450 /*
451 * Normally there is plenty of room in the current buffer for
452 * the command.
453 */
454 if (likely(qptr->index + 2 < qptr->pool_size_m1)) {
455 uint64_t *ptr =
456 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
457 base_ptr_div128 << 7);
458 ptr += qptr->index;
459 qptr->index += 2;
460 ptr[0] = cmd1;
461 ptr[1] = cmd2;
462 } else {
463 uint64_t *ptr;
464 /*
465 * Figure out how many command words will fit in this
466 * buffer. One location will be needed for the next
467 * buffer pointer.
468 */
469 int count = qptr->pool_size_m1 - qptr->index;
470 /*
471 * We need a new command buffer. Fail if there isn't
472 * one available.
473 */
474 uint64_t *new_buffer =
475 (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
476 if (unlikely(new_buffer == NULL)) {
477 if (likely(use_locking))
478 __cvmx_cmd_queue_unlock(qptr);
479 return CVMX_CMD_QUEUE_NO_MEMORY;
480 }
481 count--;
482 ptr =
483 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
484 base_ptr_div128 << 7);
485 ptr += qptr->index;
486 *ptr++ = cmd1;
487 if (likely(count))
488 *ptr++ = cmd2;
489 *ptr = cvmx_ptr_to_phys(new_buffer);
490 /*
491 * The current buffer is full and has a link to the
492 * next buffer. Time to write the rest of the commands
493 * into the new buffer.
494 */
495 qptr->base_ptr_div128 = *ptr >> 7;
496 qptr->index = 0;
497 if (unlikely(count == 0)) {
498 qptr->index = 1;
499 new_buffer[0] = cmd2;
500 }
501 }
502
503 /* All updates are complete. Release the lock and return */
504 if (likely(use_locking))
505 __cvmx_cmd_queue_unlock(qptr);
506 return CVMX_CMD_QUEUE_SUCCESS;
507}
508
509/**
510 * Simple function to write three command words to a command
511 * queue.
512 *
513 * @queue_id: Hardware command queue to write to
514 * @use_locking:
515 * Use internal locking to ensure exclusive access for queue
516 * updates. If you don't use this locking you must ensure
517 * exclusivity some other way. Locking is strongly recommended.
518 * @cmd1: Command
519 * @cmd2: Command
520 * @cmd3: Command
521 *
522 * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
523 */
524static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t
525 queue_id,
526 int use_locking,
527 uint64_t cmd1,
528 uint64_t cmd2,
529 uint64_t cmd3)
530{
531 __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
532
533 /* Make sure nobody else is updating the same queue */
534 if (likely(use_locking))
535 __cvmx_cmd_queue_lock(queue_id, qptr);
536
537 /*
538 * If a max queue length was specified then make sure we don't
539 * exceed it. If any part of the command would be below the
540 * limit we allow it.
541 */
542 if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
543 if (unlikely
544 (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
545 if (likely(use_locking))
546 __cvmx_cmd_queue_unlock(qptr);
547 return CVMX_CMD_QUEUE_FULL;
548 }
549 }
550
551 /*
552 * Normally there is plenty of room in the current buffer for
553 * the command.
554 */
555 if (likely(qptr->index + 3 < qptr->pool_size_m1)) {
556 uint64_t *ptr =
557 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
558 base_ptr_div128 << 7);
559 ptr += qptr->index;
560 qptr->index += 3;
561 ptr[0] = cmd1;
562 ptr[1] = cmd2;
563 ptr[2] = cmd3;
564 } else {
565 uint64_t *ptr;
566 /*
567 * Figure out how many command words will fit in this
568 * buffer. One location will be needed for the next
569 * buffer pointer
570 */
571 int count = qptr->pool_size_m1 - qptr->index;
572 /*
573 * We need a new command buffer. Fail if there isn't
574 * one available
575 */
576 uint64_t *new_buffer =
577 (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
578 if (unlikely(new_buffer == NULL)) {
579 if (likely(use_locking))
580 __cvmx_cmd_queue_unlock(qptr);
581 return CVMX_CMD_QUEUE_NO_MEMORY;
582 }
583 count--;
584 ptr =
585 (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
586 base_ptr_div128 << 7);
587 ptr += qptr->index;
588 *ptr++ = cmd1;
589 if (count) {
590 *ptr++ = cmd2;
591 if (count > 1)
592 *ptr++ = cmd3;
593 }
594 *ptr = cvmx_ptr_to_phys(new_buffer);
595 /*
596 * The current buffer is full and has a link to the
597 * next buffer. Time to write the rest of the commands
598 * into the new buffer.
599 */
600 qptr->base_ptr_div128 = *ptr >> 7;
601 qptr->index = 0;
602 ptr = new_buffer;
603 if (count == 0) {
604 *ptr++ = cmd2;
605 qptr->index++;
606 }
607 if (count < 2) {
608 *ptr++ = cmd3;
609 qptr->index++;
610 }
611 }
612
613 /* All updates are complete. Release the lock and return */
614 if (likely(use_locking))
615 __cvmx_cmd_queue_unlock(qptr);
616 return CVMX_CMD_QUEUE_SUCCESS;
617}
618
619#endif /* __CVMX_CMD_QUEUE_H__ */