Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Multipath support for RPC
4 *
5 * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved.
6 *
7 * Trond Myklebust <trond.myklebust@primarydata.com>
8 *
9 */
10#include <linux/types.h>
11#include <linux/kref.h>
12#include <linux/list.h>
13#include <linux/rcupdate.h>
14#include <linux/rculist.h>
15#include <linux/slab.h>
16#include <asm/cmpxchg.h>
17#include <linux/spinlock.h>
18#include <linux/sunrpc/xprt.h>
19#include <linux/sunrpc/addr.h>
20#include <linux/sunrpc/xprtmultipath.h>
21
22typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head,
23 const struct rpc_xprt *cur);
24
25static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
26static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin;
27static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall;
28
29static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
30 struct rpc_xprt *xprt)
31{
32 if (unlikely(xprt_get(xprt) == NULL))
33 return;
34 list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list);
35 smp_wmb();
36 if (xps->xps_nxprts == 0)
37 xps->xps_net = xprt->xprt_net;
38 xps->xps_nxprts++;
39}
40
41/**
42 * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch
43 * @xps: pointer to struct rpc_xprt_switch
44 * @xprt: pointer to struct rpc_xprt
45 *
46 * Adds xprt to the end of the list of struct rpc_xprt in xps.
47 */
48void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps,
49 struct rpc_xprt *xprt)
50{
51 if (xprt == NULL)
52 return;
53 spin_lock(&xps->xps_lock);
54 if ((xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) &&
55 !rpc_xprt_switch_has_addr(xps, (struct sockaddr *)&xprt->addr))
56 xprt_switch_add_xprt_locked(xps, xprt);
57 spin_unlock(&xps->xps_lock);
58}
59
60static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps,
61 struct rpc_xprt *xprt)
62{
63 if (unlikely(xprt == NULL))
64 return;
65 xps->xps_nxprts--;
66 if (xps->xps_nxprts == 0)
67 xps->xps_net = NULL;
68 smp_wmb();
69 list_del_rcu(&xprt->xprt_switch);
70}
71
72/**
73 * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch
74 * @xps: pointer to struct rpc_xprt_switch
75 * @xprt: pointer to struct rpc_xprt
76 *
77 * Removes xprt from the list of struct rpc_xprt in xps.
78 */
79void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps,
80 struct rpc_xprt *xprt)
81{
82 spin_lock(&xps->xps_lock);
83 xprt_switch_remove_xprt_locked(xps, xprt);
84 spin_unlock(&xps->xps_lock);
85 xprt_put(xprt);
86}
87
88/**
89 * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch
90 * @xprt: pointer to struct rpc_xprt
91 * @gfp_flags: allocation flags
92 *
93 * On success, returns an initialised struct rpc_xprt_switch, containing
94 * the entry xprt. Returns NULL on failure.
95 */
96struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt,
97 gfp_t gfp_flags)
98{
99 struct rpc_xprt_switch *xps;
100
101 xps = kmalloc(sizeof(*xps), gfp_flags);
102 if (xps != NULL) {
103 spin_lock_init(&xps->xps_lock);
104 kref_init(&xps->xps_kref);
105 xps->xps_nxprts = 0;
106 INIT_LIST_HEAD(&xps->xps_xprt_list);
107 xps->xps_iter_ops = &rpc_xprt_iter_singular;
108 xprt_switch_add_xprt_locked(xps, xprt);
109 }
110
111 return xps;
112}
113
114static void xprt_switch_free_entries(struct rpc_xprt_switch *xps)
115{
116 spin_lock(&xps->xps_lock);
117 while (!list_empty(&xps->xps_xprt_list)) {
118 struct rpc_xprt *xprt;
119
120 xprt = list_first_entry(&xps->xps_xprt_list,
121 struct rpc_xprt, xprt_switch);
122 xprt_switch_remove_xprt_locked(xps, xprt);
123 spin_unlock(&xps->xps_lock);
124 xprt_put(xprt);
125 spin_lock(&xps->xps_lock);
126 }
127 spin_unlock(&xps->xps_lock);
128}
129
130static void xprt_switch_free(struct kref *kref)
131{
132 struct rpc_xprt_switch *xps = container_of(kref,
133 struct rpc_xprt_switch, xps_kref);
134
135 xprt_switch_free_entries(xps);
136 kfree_rcu(xps, xps_rcu);
137}
138
139/**
140 * xprt_switch_get - Return a reference to a rpc_xprt_switch
141 * @xps: pointer to struct rpc_xprt_switch
142 *
143 * Returns a reference to xps unless the refcount is already zero.
144 */
145struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps)
146{
147 if (xps != NULL && kref_get_unless_zero(&xps->xps_kref))
148 return xps;
149 return NULL;
150}
151
152/**
153 * xprt_switch_put - Release a reference to a rpc_xprt_switch
154 * @xps: pointer to struct rpc_xprt_switch
155 *
156 * Release the reference to xps, and free it once the refcount is zero.
157 */
158void xprt_switch_put(struct rpc_xprt_switch *xps)
159{
160 if (xps != NULL)
161 kref_put(&xps->xps_kref, xprt_switch_free);
162}
163
164/**
165 * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch
166 * @xps: pointer to struct rpc_xprt_switch
167 *
168 * Sets a round-robin default policy for iterators acting on xps.
169 */
170void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps)
171{
172 if (READ_ONCE(xps->xps_iter_ops) != &rpc_xprt_iter_roundrobin)
173 WRITE_ONCE(xps->xps_iter_ops, &rpc_xprt_iter_roundrobin);
174}
175
176static
177const struct rpc_xprt_iter_ops *xprt_iter_ops(const struct rpc_xprt_iter *xpi)
178{
179 if (xpi->xpi_ops != NULL)
180 return xpi->xpi_ops;
181 return rcu_dereference(xpi->xpi_xpswitch)->xps_iter_ops;
182}
183
184static
185void xprt_iter_no_rewind(struct rpc_xprt_iter *xpi)
186{
187}
188
189static
190void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi)
191{
192 WRITE_ONCE(xpi->xpi_cursor, NULL);
193}
194
195static
196struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head)
197{
198 return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch);
199}
200
201static
202struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi)
203{
204 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
205
206 if (xps == NULL)
207 return NULL;
208 return xprt_switch_find_first_entry(&xps->xps_xprt_list);
209}
210
211static
212struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head,
213 const struct rpc_xprt *cur)
214{
215 struct rpc_xprt *pos;
216
217 list_for_each_entry_rcu(pos, head, xprt_switch) {
218 if (cur == pos)
219 return pos;
220 }
221 return NULL;
222}
223
224static
225struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi)
226{
227 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
228 struct list_head *head;
229
230 if (xps == NULL)
231 return NULL;
232 head = &xps->xps_xprt_list;
233 if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2)
234 return xprt_switch_find_first_entry(head);
235 return xprt_switch_find_current_entry(head, xpi->xpi_cursor);
236}
237
238bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps,
239 const struct sockaddr *sap)
240{
241 struct list_head *head;
242 struct rpc_xprt *pos;
243
244 if (xps == NULL || sap == NULL)
245 return false;
246
247 head = &xps->xps_xprt_list;
248 list_for_each_entry_rcu(pos, head, xprt_switch) {
249 if (rpc_cmp_addr_port(sap, (struct sockaddr *)&pos->addr)) {
250 pr_info("RPC: addr %s already in xprt switch\n",
251 pos->address_strings[RPC_DISPLAY_ADDR]);
252 return true;
253 }
254 }
255 return false;
256}
257
258static
259struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head,
260 const struct rpc_xprt *cur)
261{
262 struct rpc_xprt *pos, *prev = NULL;
263
264 list_for_each_entry_rcu(pos, head, xprt_switch) {
265 if (cur == prev)
266 return pos;
267 prev = pos;
268 }
269 return NULL;
270}
271
272static
273struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head,
274 struct rpc_xprt **cursor,
275 xprt_switch_find_xprt_t find_next)
276{
277 struct rpc_xprt *cur, *pos, *old;
278
279 cur = READ_ONCE(*cursor);
280 for (;;) {
281 old = cur;
282 pos = find_next(head, old);
283 if (pos == NULL)
284 break;
285 cur = cmpxchg_relaxed(cursor, old, pos);
286 if (cur == old)
287 break;
288 }
289 return pos;
290}
291
292static
293struct rpc_xprt *xprt_iter_next_entry_multiple(struct rpc_xprt_iter *xpi,
294 xprt_switch_find_xprt_t find_next)
295{
296 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
297
298 if (xps == NULL)
299 return NULL;
300 return xprt_switch_set_next_cursor(&xps->xps_xprt_list,
301 &xpi->xpi_cursor,
302 find_next);
303}
304
305static
306struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head,
307 const struct rpc_xprt *cur)
308{
309 struct rpc_xprt *ret;
310
311 ret = xprt_switch_find_next_entry(head, cur);
312 if (ret != NULL)
313 return ret;
314 return xprt_switch_find_first_entry(head);
315}
316
317static
318struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi)
319{
320 return xprt_iter_next_entry_multiple(xpi,
321 xprt_switch_find_next_entry_roundrobin);
322}
323
324static
325struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi)
326{
327 return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry);
328}
329
330/*
331 * xprt_iter_rewind - Resets the xprt iterator
332 * @xpi: pointer to rpc_xprt_iter
333 *
334 * Resets xpi to ensure that it points to the first entry in the list
335 * of transports.
336 */
337static
338void xprt_iter_rewind(struct rpc_xprt_iter *xpi)
339{
340 rcu_read_lock();
341 xprt_iter_ops(xpi)->xpi_rewind(xpi);
342 rcu_read_unlock();
343}
344
345static void __xprt_iter_init(struct rpc_xprt_iter *xpi,
346 struct rpc_xprt_switch *xps,
347 const struct rpc_xprt_iter_ops *ops)
348{
349 rcu_assign_pointer(xpi->xpi_xpswitch, xprt_switch_get(xps));
350 xpi->xpi_cursor = NULL;
351 xpi->xpi_ops = ops;
352}
353
354/**
355 * xprt_iter_init - Initialise an xprt iterator
356 * @xpi: pointer to rpc_xprt_iter
357 * @xps: pointer to rpc_xprt_switch
358 *
359 * Initialises the iterator to use the default iterator ops
360 * as set in xps. This function is mainly intended for internal
361 * use in the rpc_client.
362 */
363void xprt_iter_init(struct rpc_xprt_iter *xpi,
364 struct rpc_xprt_switch *xps)
365{
366 __xprt_iter_init(xpi, xps, NULL);
367}
368
369/**
370 * xprt_iter_init_listall - Initialise an xprt iterator
371 * @xpi: pointer to rpc_xprt_iter
372 * @xps: pointer to rpc_xprt_switch
373 *
374 * Initialises the iterator to iterate once through the entire list
375 * of entries in xps.
376 */
377void xprt_iter_init_listall(struct rpc_xprt_iter *xpi,
378 struct rpc_xprt_switch *xps)
379{
380 __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall);
381}
382
383/**
384 * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
385 * @xpi: pointer to rpc_xprt_iter
386 * @xps: pointer to a new rpc_xprt_switch or NULL
387 *
388 * Swaps out the existing xpi->xpi_xpswitch with a new value.
389 */
390struct rpc_xprt_switch *xprt_iter_xchg_switch(struct rpc_xprt_iter *xpi,
391 struct rpc_xprt_switch *newswitch)
392{
393 struct rpc_xprt_switch __rcu *oldswitch;
394
395 /* Atomically swap out the old xpswitch */
396 oldswitch = xchg(&xpi->xpi_xpswitch, RCU_INITIALIZER(newswitch));
397 if (newswitch != NULL)
398 xprt_iter_rewind(xpi);
399 return rcu_dereference_protected(oldswitch, true);
400}
401
402/**
403 * xprt_iter_destroy - Destroys the xprt iterator
404 * @xpi pointer to rpc_xprt_iter
405 */
406void xprt_iter_destroy(struct rpc_xprt_iter *xpi)
407{
408 xprt_switch_put(xprt_iter_xchg_switch(xpi, NULL));
409}
410
411/**
412 * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor
413 * @xpi: pointer to rpc_xprt_iter
414 *
415 * Returns a pointer to the struct rpc_xprt that is currently
416 * pointed to by the cursor.
417 * Caller must be holding rcu_read_lock().
418 */
419struct rpc_xprt *xprt_iter_xprt(struct rpc_xprt_iter *xpi)
420{
421 WARN_ON_ONCE(!rcu_read_lock_held());
422 return xprt_iter_ops(xpi)->xpi_xprt(xpi);
423}
424
425static
426struct rpc_xprt *xprt_iter_get_helper(struct rpc_xprt_iter *xpi,
427 struct rpc_xprt *(*fn)(struct rpc_xprt_iter *))
428{
429 struct rpc_xprt *ret;
430
431 do {
432 ret = fn(xpi);
433 if (ret == NULL)
434 break;
435 ret = xprt_get(ret);
436 } while (ret == NULL);
437 return ret;
438}
439
440/**
441 * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor
442 * @xpi: pointer to rpc_xprt_iter
443 *
444 * Returns a reference to the struct rpc_xprt that is currently
445 * pointed to by the cursor.
446 */
447struct rpc_xprt *xprt_iter_get_xprt(struct rpc_xprt_iter *xpi)
448{
449 struct rpc_xprt *xprt;
450
451 rcu_read_lock();
452 xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_xprt);
453 rcu_read_unlock();
454 return xprt;
455}
456
457/**
458 * xprt_iter_get_next - Returns the next rpc_xprt following the cursor
459 * @xpi: pointer to rpc_xprt_iter
460 *
461 * Returns a reference to the struct rpc_xprt that immediately follows the
462 * entry pointed to by the cursor.
463 */
464struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi)
465{
466 struct rpc_xprt *xprt;
467
468 rcu_read_lock();
469 xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_next);
470 rcu_read_unlock();
471 return xprt;
472}
473
474/* Policy for always returning the first entry in the rpc_xprt_switch */
475static
476const struct rpc_xprt_iter_ops rpc_xprt_iter_singular = {
477 .xpi_rewind = xprt_iter_no_rewind,
478 .xpi_xprt = xprt_iter_first_entry,
479 .xpi_next = xprt_iter_first_entry,
480};
481
482/* Policy for round-robin iteration of entries in the rpc_xprt_switch */
483static
484const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin = {
485 .xpi_rewind = xprt_iter_default_rewind,
486 .xpi_xprt = xprt_iter_current_entry,
487 .xpi_next = xprt_iter_next_entry_roundrobin,
488};
489
490/* Policy for once-through iteration of entries in the rpc_xprt_switch */
491static
492const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = {
493 .xpi_rewind = xprt_iter_default_rewind,
494 .xpi_xprt = xprt_iter_current_entry,
495 .xpi_next = xprt_iter_next_entry_all,
496};