1/*
  2 * Copyright 2012 Tilera Corporation. All Rights Reserved.
  3 *
  4 *   This program is free software; you can redistribute it and/or
  5 *   modify it under the terms of the GNU General Public License
  6 *   as published by the Free Software Foundation, version 2.
  7 *
  8 *   This program is distributed in the hope that it will be useful, but
  9 *   WITHOUT ANY WARRANTY; without even the implied warranty of
 10 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 11 *   NON INFRINGEMENT.  See the GNU General Public License for
 12 *   more details.
 13 */
 14
 15#ifndef _GXIO_DMA_QUEUE_H_
 16#define _GXIO_DMA_QUEUE_H_
 17
 18/*
 19 * DMA queue management APIs shared between TRIO and mPIPE.
 20 */
 21
 22#include <gxio/common.h>
 23
 24/* The credit counter lives in the high 32 bits. */
 25#define DMA_QUEUE_CREDIT_SHIFT 32
 26
 27/*
 28 * State object that tracks a DMA queue's head and tail indices, as
 29 * well as the number of commands posted and completed.  The
 30 * structure is accessed via a thread-safe, lock-free algorithm.
 31 */
 32typedef struct {
 33	/*
 34	 * Address of a MPIPE_EDMA_POST_REGION_VAL_t,
 35	 * TRIO_PUSH_DMA_REGION_VAL_t, or TRIO_PULL_DMA_REGION_VAL_t
 36	 * register.  These register have identical encodings and provide
 37	 * information about how many commands have been processed.
 38	 */
 39	void *post_region_addr;
 40
 41	/*
 42	 * A lazily-updated count of how many edescs the hardware has
 43	 * completed.
 44	 */
 45	uint64_t hw_complete_count __attribute__ ((aligned(64)));
 46
 47	/*
 48	 * High 32 bits are a count of available egress command credits,
 49	 * low 24 bits are the next egress "slot".
 50	 */
 51	int64_t credits_and_next_index;
 52
 53} __gxio_dma_queue_t;
 54
 55/* Initialize a dma queue. */
 56extern void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue,
 57				  void *post_region_addr,
 58				  unsigned int num_entries);
 59
 60/*
 61 * Update the "credits_and_next_index" and "hw_complete_count" fields
 62 * based on pending hardware completions.  Note that some other thread
 63 * may have already done this and, importantly, may still be in the
 64 * process of updating "credits_and_next_index".
 65 */
 66extern void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue);
 67
 68/* Wait for credits to become available. */
 69extern int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue,
 70						 int64_t modifier);
 71
 72/* Reserve slots in the queue, optionally waiting for slots to become
 73 * available, and optionally returning a "completion_slot" suitable for
 74 * direct comparison to "hw_complete_count".
 75 */
 76static inline int64_t __gxio_dma_queue_reserve(__gxio_dma_queue_t *dma_queue,
 77					       unsigned int num, bool wait,
 78					       bool completion)
 79{
 80	uint64_t slot;
 81
 82	/*
 83	 * Try to reserve 'num' egress command slots.  We do this by
 84	 * constructing a constant that subtracts N credits and adds N to
 85	 * the index, and using fetchaddgez to only apply it if the credits
 86	 * count doesn't go negative.
 87	 */
 88	int64_t modifier = (((int64_t)(-num)) << DMA_QUEUE_CREDIT_SHIFT) | num;
 89	int64_t old =
 90		__insn_fetchaddgez(&dma_queue->credits_and_next_index,
 91				   modifier);
 92
 93	if (unlikely(old + modifier < 0)) {
 94		/*
 95		 * We're out of credits.  Try once to get more by checking for
 96		 * completed egress commands.  If that fails, wait or fail.
 97		 */
 98		__gxio_dma_queue_update_credits(dma_queue);
 99		old = __insn_fetchaddgez(&dma_queue->credits_and_next_index,
100					 modifier);
101		if (old + modifier < 0) {
102			if (wait)
103				old = __gxio_dma_queue_wait_for_credits
104					(dma_queue, modifier);
105			else
106				return GXIO_ERR_DMA_CREDITS;
107		}
108	}
109
110	/* The bottom 24 bits of old encode the "slot". */
111	slot = (old & 0xffffff);
112
113	if (completion) {
114		/*
115		 * A "completion_slot" is a "slot" which can be compared to
116		 * "hw_complete_count" at any time in the future.  To convert
117		 * "slot" into a "completion_slot", we access "hw_complete_count"
118		 * once (knowing that we have reserved a slot, and thus, it will
119		 * be "basically" accurate), and combine its high 40 bits with
120		 * the 24 bit "slot", and handle "wrapping" by adding "1 << 24"
121		 * if the result is LESS than "hw_complete_count".
122		 */
123		uint64_t complete;
124		complete = ACCESS_ONCE(dma_queue->hw_complete_count);
125		slot |= (complete & 0xffffffffff000000);
126		if (slot < complete)
127			slot += 0x1000000;
128	}
129
130	/*
131	 * If any of our slots mod 256 were equivalent to 0, go ahead and
132	 * collect some egress credits, and update "hw_complete_count", and
133	 * make sure the index doesn't overflow into the credits.
134	 */
135	if (unlikely(((old + num) & 0xff) < num)) {
136		__gxio_dma_queue_update_credits(dma_queue);
137
138		/* Make sure the index doesn't overflow into the credits. */
139#ifdef __BIG_ENDIAN__
140		*(((uint8_t *)&dma_queue->credits_and_next_index) + 4) = 0;
141#else
142		*(((uint8_t *)&dma_queue->credits_and_next_index) + 3) = 0;
143#endif
144	}
145
146	return slot;
147}
148
149/* Non-inlinable "__gxio_dma_queue_reserve(..., true)". */
150extern int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue,
151					    unsigned int num, int wait);
152
153/* Check whether a particular "completion slot" has completed.
154 *
155 * Note that this function requires a "completion slot", and thus
156 * cannot be used with the result of any "reserve_fast" function.
157 */
158extern int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue,
159					int64_t completion_slot, int update);
160
161#endif /* !_GXIO_DMA_QUEUE_H_ */