1/*
  2 * Copyright 2012 Tilera Corporation. All Rights Reserved.
  3 *
  4 *   This program is free software; you can redistribute it and/or
  5 *   modify it under the terms of the GNU General Public License
  6 *   as published by the Free Software Foundation, version 2.
  7 *
  8 *   This program is distributed in the hope that it will be useful, but
  9 *   WITHOUT ANY WARRANTY; without even the implied warranty of
 10 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 11 *   NON INFRINGEMENT.  See the GNU General Public License for
 12 *   more details.
 13 */
 14
 15#include <linux/io.h>
 16#include <linux/atomic.h>
 17#include <linux/module.h>
 18#include <gxio/dma_queue.h>
 19
 20/* Wait for a memory read to complete. */
 21#define wait_for_value(val)                             \
 22  __asm__ __volatile__("move %0, %0" :: "r"(val))
 23
 24/* The index is in the low 16. */
 25#define DMA_QUEUE_INDEX_MASK ((1 << 16) - 1)
 26
 27/*
 28 * The hardware descriptor-ring type.
 29 * This matches the types used by mpipe (MPIPE_EDMA_POST_REGION_VAL_t)
 30 * and trio (TRIO_PUSH_DMA_REGION_VAL_t or TRIO_PULL_DMA_REGION_VAL_t).
 31 * See those types for more documentation on the individual fields.
 32 */
 33typedef union {
 34	struct {
 35#ifndef __BIG_ENDIAN__
 36		uint64_t ring_idx:16;
 37		uint64_t count:16;
 38		uint64_t gen:1;
 39		uint64_t __reserved:31;
 40#else
 41		uint64_t __reserved:31;
 42		uint64_t gen:1;
 43		uint64_t count:16;
 44		uint64_t ring_idx:16;
 45#endif
 46	};
 47	uint64_t word;
 48} __gxio_ring_t;
 49
 50void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue,
 51			   void *post_region_addr, unsigned int num_entries)
 52{
 53	/*
 54	 * Limit 65536 entry rings to 65535 credits because we only have a
 55	 * 16 bit completion counter.
 56	 */
 57	int64_t credits = (num_entries < 65536) ? num_entries : 65535;
 58
 59	memset(dma_queue, 0, sizeof(*dma_queue));
 60
 61	dma_queue->post_region_addr = post_region_addr;
 62	dma_queue->hw_complete_count = 0;
 63	dma_queue->credits_and_next_index = credits << DMA_QUEUE_CREDIT_SHIFT;
 64}
 65
 66EXPORT_SYMBOL_GPL(__gxio_dma_queue_init);
 67
 68void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue)
 69{
 70	__gxio_ring_t val;
 71	uint64_t count;
 72	uint64_t delta;
 73	uint64_t new_count;
 74
 75	/*
 76	 * Read the 64-bit completion count without touching the cache, so
 77	 * we later avoid having to evict any sharers of this cache line
 78	 * when we update it below.
 79	 */
 80	uint64_t orig_hw_complete_count =
 81		cmpxchg(&dma_queue->hw_complete_count,
 82			-1, -1);
 83
 84	/* Make sure the load completes before we access the hardware. */
 85	wait_for_value(orig_hw_complete_count);
 86
 87	/* Read the 16-bit count of how many packets it has completed. */
 88	val.word = __gxio_mmio_read(dma_queue->post_region_addr);
 89	count = val.count;
 90
 91	/*
 92	 * Calculate the number of completions since we last updated the
 93	 * 64-bit counter.  It's safe to ignore the high bits because the
 94	 * maximum credit value is 65535.
 95	 */
 96	delta = (count - orig_hw_complete_count) & 0xffff;
 97	if (delta == 0)
 98		return;
 99
100	/*
101	 * Try to write back the count, advanced by delta.  If we race with
102	 * another thread, this might fail, in which case we return
103	 * immediately on the assumption that some credits are (or at least
104	 * were) available.
105	 */
106	new_count = orig_hw_complete_count + delta;
107	if (cmpxchg(&dma_queue->hw_complete_count,
108		    orig_hw_complete_count,
109		    new_count) != orig_hw_complete_count)
110		return;
111
112	/*
113	 * We succeeded in advancing the completion count; add back the
114	 * corresponding number of egress credits.
115	 */
116	__insn_fetchadd(&dma_queue->credits_and_next_index,
117			(delta << DMA_QUEUE_CREDIT_SHIFT));
118}
119
120EXPORT_SYMBOL_GPL(__gxio_dma_queue_update_credits);
121
122/*
123 * A separate 'blocked' method for put() so that backtraces and
124 * profiles will clearly indicate that we're wasting time spinning on
125 * egress availability rather than actually posting commands.
126 */
127int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue,
128					  int64_t modifier)
129{
130	int backoff = 16;
131	int64_t old;
132
133	do {
134		int i;
135		/* Back off to avoid spamming memory networks. */
136		for (i = backoff; i > 0; i--)
137			__insn_mfspr(SPR_PASS);
138
139		/* Check credits again. */
140		__gxio_dma_queue_update_credits(dma_queue);
141		old = __insn_fetchaddgez(&dma_queue->credits_and_next_index,
142					 modifier);
143
144		/* Calculate bounded exponential backoff for next iteration. */
145		if (backoff < 256)
146			backoff *= 2;
147	} while (old + modifier < 0);
148
149	return old;
150}
151
152EXPORT_SYMBOL_GPL(__gxio_dma_queue_wait_for_credits);
153
154int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue,
155				     unsigned int num, int wait)
156{
157	return __gxio_dma_queue_reserve(dma_queue, num, wait != 0, true);
158}
159
160EXPORT_SYMBOL_GPL(__gxio_dma_queue_reserve_aux);
161
162int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue,
163				 int64_t completion_slot, int update)
164{
165	if (update) {
166		if (ACCESS_ONCE(dma_queue->hw_complete_count) >
167		    completion_slot)
168			return 1;
169
170		__gxio_dma_queue_update_credits(dma_queue);
171	}
172
173	return ACCESS_ONCE(dma_queue->hw_complete_count) > completion_slot;
174}
175
176EXPORT_SYMBOL_GPL(__gxio_dma_queue_is_complete);