1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2023 Intel Corporation
  4 */
  5
  6#include <linux/dma-fence.h>
  7#include <linux/interval_tree_generic.h>
  8#include <linux/slab.h>
  9
 10#include "xe_macros.h"
 11#include "xe_range_fence.h"
 12
 13#define XE_RANGE_TREE_START(_node)	((_node)->start)
 14#define XE_RANGE_TREE_LAST(_node)	((_node)->last)
 15
 16INTERVAL_TREE_DEFINE(struct xe_range_fence, rb, u64, __subtree_last,
 17		     XE_RANGE_TREE_START, XE_RANGE_TREE_LAST, static,
 18		     xe_range_fence_tree);
 19
 20static void
 21xe_range_fence_signal_notify(struct dma_fence *fence, struct dma_fence_cb *cb)
 22{
 23	struct xe_range_fence *rfence = container_of(cb, typeof(*rfence), cb);
 24	struct xe_range_fence_tree *tree = rfence->tree;
 25
 26	llist_add(&rfence->link, &tree->list);
 27}
 28
 29static bool __xe_range_fence_tree_cleanup(struct xe_range_fence_tree *tree)
 30{
 31	struct llist_node *node = llist_del_all(&tree->list);
 32	struct xe_range_fence *rfence, *next;
 33
 34	llist_for_each_entry_safe(rfence, next, node, link) {
 35		xe_range_fence_tree_remove(rfence, &tree->root);
 36		dma_fence_put(rfence->fence);
 37		kfree(rfence);
 38	}
 39
 40	return !!node;
 41}
 42
 43/**
 44 * xe_range_fence_insert() - range fence insert
 45 * @tree: range fence tree to insert intoi
 46 * @rfence: range fence
 47 * @ops: range fence ops
 48 * @start: start address of range fence
 49 * @last: last address of range fence
 50 * @fence: dma fence which signals range fence can be removed + freed
 51 *
 52 * Return: 0 on success, non-zero on failure
 53 */
 54int xe_range_fence_insert(struct xe_range_fence_tree *tree,
 55			  struct xe_range_fence *rfence,
 56			  const struct xe_range_fence_ops *ops,
 57			  u64 start, u64 last, struct dma_fence *fence)
 58{
 59	int err = 0;
 60
 61	__xe_range_fence_tree_cleanup(tree);
 62
 63	if (dma_fence_is_signaled(fence))
 64		goto free;
 65
 66	rfence->ops = ops;
 67	rfence->start = start;
 68	rfence->last = last;
 69	rfence->tree = tree;
 70	rfence->fence = dma_fence_get(fence);
 71	err = dma_fence_add_callback(fence, &rfence->cb,
 72				     xe_range_fence_signal_notify);
 73	if (err == -ENOENT) {
 74		dma_fence_put(fence);
 75		err = 0;
 76		goto free;
 77	} else if (err == 0) {
 78		xe_range_fence_tree_insert(rfence, &tree->root);
 79		return 0;
 80	}
 81
 82free:
 83	if (ops->free)
 84		ops->free(rfence);
 85
 86	return err;
 87}
 88
 89static void xe_range_fence_tree_remove_all(struct xe_range_fence_tree *tree)
 90{
 91	struct xe_range_fence *rfence;
 92	bool retry = true;
 93
 94	rfence = xe_range_fence_tree_iter_first(&tree->root, 0, U64_MAX);
 95	while (rfence) {
 96		/* Should be ok with the minimalistic callback */
 97		if (dma_fence_remove_callback(rfence->fence, &rfence->cb))
 98			llist_add(&rfence->link, &tree->list);
 99		rfence = xe_range_fence_tree_iter_next(rfence, 0, U64_MAX);
100	}
101
102	while (retry)
103		retry = __xe_range_fence_tree_cleanup(tree);
104}
105
106/**
107 * xe_range_fence_tree_init() - Init range fence tree
108 * @tree: range fence tree
109 */
110void xe_range_fence_tree_init(struct xe_range_fence_tree *tree)
111{
112	memset(tree, 0, sizeof(*tree));
113}
114
115/**
116 * xe_range_fence_tree_fini() - Fini range fence tree
117 * @tree: range fence tree
118 */
119void xe_range_fence_tree_fini(struct xe_range_fence_tree *tree)
120{
121	xe_range_fence_tree_remove_all(tree);
122	XE_WARN_ON(!RB_EMPTY_ROOT(&tree->root.rb_root));
123}
124
125/**
126 * xe_range_fence_tree_first() - range fence tree iterator first
127 * @tree: range fence tree
128 * @start: start address of range fence
129 * @last: last address of range fence
130 *
131 * Return: first range fence found in range or NULL
132 */
133struct xe_range_fence *
134xe_range_fence_tree_first(struct xe_range_fence_tree *tree, u64 start,
135			  u64 last)
136{
137	return xe_range_fence_tree_iter_first(&tree->root, start, last);
138}
139
140/**
141 * xe_range_fence_tree_next() - range fence tree iterator next
142 * @rfence: current range fence
143 * @start: start address of range fence
144 * @last: last address of range fence
145 *
146 * Return: next range fence found in range or NULL
147 */
148struct xe_range_fence *
149xe_range_fence_tree_next(struct xe_range_fence *rfence, u64 start, u64 last)
150{
151	return xe_range_fence_tree_iter_next(rfence, start, last);
152}
153
154static void xe_range_fence_free(struct xe_range_fence *rfence)
155{
156	kfree(rfence);
157}
158
159const struct xe_range_fence_ops xe_range_fence_kfree_ops = {
160	.free = xe_range_fence_free,
161};