1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2019 Intel Corporation
  5 */
  6
  7#ifndef INTEL_RING_H
  8#define INTEL_RING_H
  9
 10#include "i915_gem.h" /* GEM_BUG_ON */
 11#include "i915_request.h"
 12#include "intel_ring_types.h"
 13
 14struct intel_engine_cs;
 15
 16struct intel_ring *
 17intel_engine_create_ring(struct intel_engine_cs *engine, int size);
 18
 19u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords);
 20int intel_ring_cacheline_align(struct i915_request *rq);
 21
 22unsigned int intel_ring_update_space(struct intel_ring *ring);
 23
 24int intel_ring_pin(struct intel_ring *ring);
 25void intel_ring_unpin(struct intel_ring *ring);
 26void intel_ring_reset(struct intel_ring *ring, u32 tail);
 27
 28void intel_ring_free(struct kref *ref);
 29
 30static inline struct intel_ring *intel_ring_get(struct intel_ring *ring)
 31{
 32	kref_get(&ring->ref);
 33	return ring;
 34}
 35
 36static inline void intel_ring_put(struct intel_ring *ring)
 37{
 38	kref_put(&ring->ref, intel_ring_free);
 39}
 40
 41static inline void intel_ring_advance(struct i915_request *rq, u32 *cs)
 42{
 43	/* Dummy function.
 44	 *
 45	 * This serves as a placeholder in the code so that the reader
 46	 * can compare against the preceding intel_ring_begin() and
 47	 * check that the number of dwords emitted matches the space
 48	 * reserved for the command packet (i.e. the value passed to
 49	 * intel_ring_begin()).
 50	 */
 51	GEM_BUG_ON((rq->ring->vaddr + rq->ring->emit) != cs);
 52}
 53
 54static inline u32 intel_ring_wrap(const struct intel_ring *ring, u32 pos)
 55{
 56	return pos & (ring->size - 1);
 57}
 58
 59static inline int intel_ring_direction(const struct intel_ring *ring,
 60				       u32 next, u32 prev)
 61{
 62	typecheck(typeof(ring->size), next);
 63	typecheck(typeof(ring->size), prev);
 64	return (next - prev) << ring->wrap;
 65}
 66
 67static inline bool
 68intel_ring_offset_valid(const struct intel_ring *ring,
 69			unsigned int pos)
 70{
 71	if (pos & -ring->size) /* must be strictly within the ring */
 72		return false;
 73
 74	if (!IS_ALIGNED(pos, 8)) /* must be qword aligned */
 75		return false;
 76
 77	return true;
 78}
 79
 80static inline u32 intel_ring_offset(const struct i915_request *rq, void *addr)
 81{
 82	/* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
 83	u32 offset = addr - rq->ring->vaddr;
 84	GEM_BUG_ON(offset > rq->ring->size);
 85	return intel_ring_wrap(rq->ring, offset);
 86}
 87
 88static inline void
 89assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail)
 90{
 91	unsigned int head = READ_ONCE(ring->head);
 92
 93	GEM_BUG_ON(!intel_ring_offset_valid(ring, tail));
 94
 95	/*
 96	 * "Ring Buffer Use"
 97	 *	Gen2 BSpec "1. Programming Environment" / 1.4.4.6
 98	 *	Gen3 BSpec "1c Memory Interface Functions" / 2.3.4.5
 99	 *	Gen4+ BSpec "1c Memory Interface and Command Stream" / 5.3.4.5
100	 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
101	 * same cacheline, the Head Pointer must not be greater than the Tail
102	 * Pointer."
103	 *
104	 * We use ring->head as the last known location of the actual RING_HEAD,
105	 * it may have advanced but in the worst case it is equally the same
106	 * as ring->head and so we should never program RING_TAIL to advance
107	 * into the same cacheline as ring->head.
108	 */
109#define cacheline(a) round_down(a, CACHELINE_BYTES)
110	GEM_BUG_ON(cacheline(tail) == cacheline(head) && tail < head);
111#undef cacheline
112}
113
114static inline unsigned int
115intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
116{
117	/* Whilst writes to the tail are strictly order, there is no
118	 * serialisation between readers and the writers. The tail may be
119	 * read by i915_request_retire() just as it is being updated
120	 * by execlists, as although the breadcrumb is complete, the context
121	 * switch hasn't been seen.
122	 */
123	assert_ring_tail_valid(ring, tail);
124	ring->tail = tail;
125	return tail;
126}
127
128static inline unsigned int
129__intel_ring_space(unsigned int head, unsigned int tail, unsigned int size)
130{
131	/*
132	 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
133	 * same cacheline, the Head Pointer must not be greater than the Tail
134	 * Pointer."
135	 */
136	GEM_BUG_ON(!is_power_of_2(size));
137	return (head - tail - CACHELINE_BYTES) & (size - 1);
138}
139
140#endif /* INTEL_RING_H */