1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2/*
  3 * Copyright (c) 2024 Oracle.  All rights reserved.
  4 */
  5
  6/* #include <linux/module.h>
  7#include <linux/slab.h> */
  8#include <linux/xarray.h>
  9#include <linux/types.h>
 10#include <linux/kref.h>
 11#include <linux/completion.h>
 12
 13#include <linux/sunrpc/svc_rdma.h>
 14#include <linux/sunrpc/rdma_rn.h>
 15
 16#include "xprt_rdma.h"
 17#include <trace/events/rpcrdma.h>
 18
 19/* Per-ib_device private data for rpcrdma */
 20struct rpcrdma_device {
 21	struct kref		rd_kref;
 22	unsigned long		rd_flags;
 23	struct ib_device	*rd_device;
 24	struct xarray		rd_xa;
 25	struct completion	rd_done;
 26};
 27
 28#define RPCRDMA_RD_F_REMOVING	(0)
 29
 30static struct ib_client rpcrdma_ib_client;
 31
 32/*
 33 * Listeners have no associated device, so we never register them.
 34 * Note that ib_get_client_data() does not check if @device is
 35 * NULL for us.
 36 */
 37static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device)
 38{
 39	if (!device)
 40		return NULL;
 41	return ib_get_client_data(device, &rpcrdma_ib_client);
 42}
 43
 44/**
 45 * rpcrdma_rn_register - register to get device removal notifications
 46 * @device: device to monitor
 47 * @rn: notification object that wishes to be notified
 48 * @done: callback to notify caller of device removal
 49 *
 50 * Returns zero on success. The callback in rn_done is guaranteed
 51 * to be invoked when the device is removed, unless this notification
 52 * is unregistered first.
 53 *
 54 * On failure, a negative errno is returned.
 55 */
 56int rpcrdma_rn_register(struct ib_device *device,
 57			struct rpcrdma_notification *rn,
 58			void (*done)(struct rpcrdma_notification *rn))
 59{
 60	struct rpcrdma_device *rd = rpcrdma_get_client_data(device);
 61
 62	if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags))
 63		return -ENETUNREACH;
 64
 65	if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0)
 66		return -ENOMEM;
 67	kref_get(&rd->rd_kref);
 68	rn->rn_done = done;
 69	trace_rpcrdma_client_register(device, rn);
 70	return 0;
 71}
 72
 73static void rpcrdma_rn_release(struct kref *kref)
 74{
 75	struct rpcrdma_device *rd = container_of(kref, struct rpcrdma_device,
 76						 rd_kref);
 77
 78	trace_rpcrdma_client_completion(rd->rd_device);
 79	complete(&rd->rd_done);
 80}
 81
 82/**
 83 * rpcrdma_rn_unregister - stop device removal notifications
 84 * @device: monitored device
 85 * @rn: notification object that no longer wishes to be notified
 86 */
 87void rpcrdma_rn_unregister(struct ib_device *device,
 88			   struct rpcrdma_notification *rn)
 89{
 90	struct rpcrdma_device *rd = rpcrdma_get_client_data(device);
 91
 92	if (!rd)
 93		return;
 94
 95	trace_rpcrdma_client_unregister(device, rn);
 96	xa_erase(&rd->rd_xa, rn->rn_index);
 97	kref_put(&rd->rd_kref, rpcrdma_rn_release);
 98}
 99
100/**
101 * rpcrdma_add_one - ib_client device insertion callback
102 * @device: device about to be inserted
103 *
104 * Returns zero on success. xprtrdma private data has been allocated
105 * for this device. On failure, a negative errno is returned.
106 */
107static int rpcrdma_add_one(struct ib_device *device)
108{
109	struct rpcrdma_device *rd;
110
111	rd = kzalloc(sizeof(*rd), GFP_KERNEL);
112	if (!rd)
113		return -ENOMEM;
114
115	kref_init(&rd->rd_kref);
116	xa_init_flags(&rd->rd_xa, XA_FLAGS_ALLOC);
117	rd->rd_device = device;
118	init_completion(&rd->rd_done);
119	ib_set_client_data(device, &rpcrdma_ib_client, rd);
120
121	trace_rpcrdma_client_add_one(device);
122	return 0;
123}
124
125/**
126 * rpcrdma_remove_one - ib_client device removal callback
127 * @device: device about to be removed
128 * @client_data: this module's private per-device data
129 *
130 * Upon return, all transports associated with @device have divested
131 * themselves from IB hardware resources.
132 */
133static void rpcrdma_remove_one(struct ib_device *device,
134			       void *client_data)
135{
136	struct rpcrdma_device *rd = client_data;
137	struct rpcrdma_notification *rn;
138	unsigned long index;
139
140	trace_rpcrdma_client_remove_one(device);
141
142	set_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags);
143	xa_for_each(&rd->rd_xa, index, rn)
144		rn->rn_done(rn);
145
146	/*
147	 * Wait only if there are still outstanding notification
148	 * registrants for this device.
149	 */
150	if (!refcount_dec_and_test(&rd->rd_kref.refcount)) {
151		trace_rpcrdma_client_wait_on(device);
152		wait_for_completion(&rd->rd_done);
153	}
154
155	trace_rpcrdma_client_remove_one_done(device);
156	xa_destroy(&rd->rd_xa);
157	kfree(rd);
158}
159
160static struct ib_client rpcrdma_ib_client = {
161	.name		= "rpcrdma",
162	.add		= rpcrdma_add_one,
163	.remove		= rpcrdma_remove_one,
164};
165
166/**
167 * rpcrdma_ib_client_unregister - unregister ib_client for xprtrdma
168 *
169 * cel: watch for orphaned rpcrdma_device objects on module unload
170 */
171void rpcrdma_ib_client_unregister(void)
172{
173	ib_unregister_client(&rpcrdma_ib_client);
174}
175
176/**
177 * rpcrdma_ib_client_register - register ib_client for rpcrdma
178 *
179 * Returns zero on success, or a negative errno.
180 */
181int rpcrdma_ib_client_register(void)
182{
183	return ib_register_client(&rpcrdma_ib_client);
184}