Loading...
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/ceph/ceph_debug.h>
3
4#include <linux/module.h>
5#include <linux/types.h>
6#include <linux/slab.h>
7#include <linux/random.h>
8#include <linux/sched.h>
9
10#include <linux/ceph/ceph_features.h>
11#include <linux/ceph/mon_client.h>
12#include <linux/ceph/libceph.h>
13#include <linux/ceph/debugfs.h>
14#include <linux/ceph/decode.h>
15#include <linux/ceph/auth.h>
16
17/*
18 * Interact with Ceph monitor cluster. Handle requests for new map
19 * versions, and periodically resend as needed. Also implement
20 * statfs() and umount().
21 *
22 * A small cluster of Ceph "monitors" are responsible for managing critical
23 * cluster configuration and state information. An odd number (e.g., 3, 5)
24 * of cmon daemons use a modified version of the Paxos part-time parliament
25 * algorithm to manage the MDS map (mds cluster membership), OSD map, and
26 * list of clients who have mounted the file system.
27 *
28 * We maintain an open, active session with a monitor at all times in order to
29 * receive timely MDSMap updates. We periodically send a keepalive byte on the
30 * TCP socket to ensure we detect a failure. If the connection does break, we
31 * randomly hunt for a new monitor. Once the connection is reestablished, we
32 * resend any outstanding requests.
33 */
34
35static const struct ceph_connection_operations mon_con_ops;
36
37static int __validate_auth(struct ceph_mon_client *monc);
38
39static int decode_mon_info(void **p, void *end, bool msgr2,
40 struct ceph_entity_addr *addr)
41{
42 void *mon_info_end;
43 u32 struct_len;
44 u8 struct_v;
45 int ret;
46
47 ret = ceph_start_decoding(p, end, 1, "mon_info_t", &struct_v,
48 &struct_len);
49 if (ret)
50 return ret;
51
52 mon_info_end = *p + struct_len;
53 ceph_decode_skip_string(p, end, e_inval); /* skip mon name */
54 ret = ceph_decode_entity_addrvec(p, end, msgr2, addr);
55 if (ret)
56 return ret;
57
58 *p = mon_info_end;
59 return 0;
60
61e_inval:
62 return -EINVAL;
63}
64
65/*
66 * Decode a monmap blob (e.g., during mount).
67 *
68 * Assume MonMap v3 (i.e. encoding with MONNAMES and MONENC).
69 */
70static struct ceph_monmap *ceph_monmap_decode(void **p, void *end, bool msgr2)
71{
72 struct ceph_monmap *monmap = NULL;
73 struct ceph_fsid fsid;
74 u32 struct_len;
75 int blob_len;
76 int num_mon;
77 u8 struct_v;
78 u32 epoch;
79 int ret;
80 int i;
81
82 ceph_decode_32_safe(p, end, blob_len, e_inval);
83 ceph_decode_need(p, end, blob_len, e_inval);
84
85 ret = ceph_start_decoding(p, end, 6, "monmap", &struct_v, &struct_len);
86 if (ret)
87 goto fail;
88
89 dout("%s struct_v %d\n", __func__, struct_v);
90 ceph_decode_copy_safe(p, end, &fsid, sizeof(fsid), e_inval);
91 ceph_decode_32_safe(p, end, epoch, e_inval);
92 if (struct_v >= 6) {
93 u32 feat_struct_len;
94 u8 feat_struct_v;
95
96 *p += sizeof(struct ceph_timespec); /* skip last_changed */
97 *p += sizeof(struct ceph_timespec); /* skip created */
98
99 ret = ceph_start_decoding(p, end, 1, "mon_feature_t",
100 &feat_struct_v, &feat_struct_len);
101 if (ret)
102 goto fail;
103
104 *p += feat_struct_len; /* skip persistent_features */
105
106 ret = ceph_start_decoding(p, end, 1, "mon_feature_t",
107 &feat_struct_v, &feat_struct_len);
108 if (ret)
109 goto fail;
110
111 *p += feat_struct_len; /* skip optional_features */
112 }
113 ceph_decode_32_safe(p, end, num_mon, e_inval);
114
115 dout("%s fsid %pU epoch %u num_mon %d\n", __func__, &fsid, epoch,
116 num_mon);
117 if (num_mon > CEPH_MAX_MON)
118 goto e_inval;
119
120 monmap = kmalloc(struct_size(monmap, mon_inst, num_mon), GFP_NOIO);
121 if (!monmap) {
122 ret = -ENOMEM;
123 goto fail;
124 }
125 monmap->fsid = fsid;
126 monmap->epoch = epoch;
127 monmap->num_mon = num_mon;
128
129 /* legacy_mon_addr map or mon_info map */
130 for (i = 0; i < num_mon; i++) {
131 struct ceph_entity_inst *inst = &monmap->mon_inst[i];
132
133 ceph_decode_skip_string(p, end, e_inval); /* skip mon name */
134 inst->name.type = CEPH_ENTITY_TYPE_MON;
135 inst->name.num = cpu_to_le64(i);
136
137 if (struct_v >= 6)
138 ret = decode_mon_info(p, end, msgr2, &inst->addr);
139 else
140 ret = ceph_decode_entity_addr(p, end, &inst->addr);
141 if (ret)
142 goto fail;
143
144 dout("%s mon%d addr %s\n", __func__, i,
145 ceph_pr_addr(&inst->addr));
146 }
147
148 return monmap;
149
150e_inval:
151 ret = -EINVAL;
152fail:
153 kfree(monmap);
154 return ERR_PTR(ret);
155}
156
157/*
158 * return true if *addr is included in the monmap.
159 */
160int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr)
161{
162 int i;
163
164 for (i = 0; i < m->num_mon; i++) {
165 if (ceph_addr_equal_no_type(addr, &m->mon_inst[i].addr))
166 return 1;
167 }
168
169 return 0;
170}
171
172/*
173 * Send an auth request.
174 */
175static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len)
176{
177 monc->pending_auth = 1;
178 monc->m_auth->front.iov_len = len;
179 monc->m_auth->hdr.front_len = cpu_to_le32(len);
180 ceph_msg_revoke(monc->m_auth);
181 ceph_msg_get(monc->m_auth); /* keep our ref */
182 ceph_con_send(&monc->con, monc->m_auth);
183}
184
185/*
186 * Close monitor session, if any.
187 */
188static void __close_session(struct ceph_mon_client *monc)
189{
190 dout("__close_session closing mon%d\n", monc->cur_mon);
191 ceph_msg_revoke(monc->m_auth);
192 ceph_msg_revoke_incoming(monc->m_auth_reply);
193 ceph_msg_revoke(monc->m_subscribe);
194 ceph_msg_revoke_incoming(monc->m_subscribe_ack);
195 ceph_con_close(&monc->con);
196
197 monc->pending_auth = 0;
198 ceph_auth_reset(monc->auth);
199}
200
201/*
202 * Pick a new monitor at random and set cur_mon. If we are repicking
203 * (i.e. cur_mon is already set), be sure to pick a different one.
204 */
205static void pick_new_mon(struct ceph_mon_client *monc)
206{
207 int old_mon = monc->cur_mon;
208
209 BUG_ON(monc->monmap->num_mon < 1);
210
211 if (monc->monmap->num_mon == 1) {
212 monc->cur_mon = 0;
213 } else {
214 int max = monc->monmap->num_mon;
215 int o = -1;
216 int n;
217
218 if (monc->cur_mon >= 0) {
219 if (monc->cur_mon < monc->monmap->num_mon)
220 o = monc->cur_mon;
221 if (o >= 0)
222 max--;
223 }
224
225 n = get_random_u32_below(max);
226 if (o >= 0 && n >= o)
227 n++;
228
229 monc->cur_mon = n;
230 }
231
232 dout("%s mon%d -> mon%d out of %d mons\n", __func__, old_mon,
233 monc->cur_mon, monc->monmap->num_mon);
234}
235
236/*
237 * Open a session with a new monitor.
238 */
239static void __open_session(struct ceph_mon_client *monc)
240{
241 int ret;
242
243 pick_new_mon(monc);
244
245 monc->hunting = true;
246 if (monc->had_a_connection) {
247 monc->hunt_mult *= CEPH_MONC_HUNT_BACKOFF;
248 if (monc->hunt_mult > CEPH_MONC_HUNT_MAX_MULT)
249 monc->hunt_mult = CEPH_MONC_HUNT_MAX_MULT;
250 }
251
252 monc->sub_renew_after = jiffies; /* i.e., expired */
253 monc->sub_renew_sent = 0;
254
255 dout("%s opening mon%d\n", __func__, monc->cur_mon);
256 ceph_con_open(&monc->con, CEPH_ENTITY_TYPE_MON, monc->cur_mon,
257 &monc->monmap->mon_inst[monc->cur_mon].addr);
258
259 /*
260 * Queue a keepalive to ensure that in case of an early fault
261 * the messenger doesn't put us into STANDBY state and instead
262 * retries. This also ensures that our timestamp is valid by
263 * the time we finish hunting and delayed_work() checks it.
264 */
265 ceph_con_keepalive(&monc->con);
266 if (ceph_msgr2(monc->client)) {
267 monc->pending_auth = 1;
268 return;
269 }
270
271 /* initiate authentication handshake */
272 ret = ceph_auth_build_hello(monc->auth,
273 monc->m_auth->front.iov_base,
274 monc->m_auth->front_alloc_len);
275 BUG_ON(ret <= 0);
276 __send_prepared_auth_request(monc, ret);
277}
278
279static void reopen_session(struct ceph_mon_client *monc)
280{
281 if (!monc->hunting)
282 pr_info("mon%d %s session lost, hunting for new mon\n",
283 monc->cur_mon, ceph_pr_addr(&monc->con.peer_addr));
284
285 __close_session(monc);
286 __open_session(monc);
287}
288
289void ceph_monc_reopen_session(struct ceph_mon_client *monc)
290{
291 mutex_lock(&monc->mutex);
292 reopen_session(monc);
293 mutex_unlock(&monc->mutex);
294}
295
296static void un_backoff(struct ceph_mon_client *monc)
297{
298 monc->hunt_mult /= 2; /* reduce by 50% */
299 if (monc->hunt_mult < 1)
300 monc->hunt_mult = 1;
301 dout("%s hunt_mult now %d\n", __func__, monc->hunt_mult);
302}
303
304/*
305 * Reschedule delayed work timer.
306 */
307static void __schedule_delayed(struct ceph_mon_client *monc)
308{
309 unsigned long delay;
310
311 if (monc->hunting)
312 delay = CEPH_MONC_HUNT_INTERVAL * monc->hunt_mult;
313 else
314 delay = CEPH_MONC_PING_INTERVAL;
315
316 dout("__schedule_delayed after %lu\n", delay);
317 mod_delayed_work(system_wq, &monc->delayed_work,
318 round_jiffies_relative(delay));
319}
320
321const char *ceph_sub_str[] = {
322 [CEPH_SUB_MONMAP] = "monmap",
323 [CEPH_SUB_OSDMAP] = "osdmap",
324 [CEPH_SUB_FSMAP] = "fsmap.user",
325 [CEPH_SUB_MDSMAP] = "mdsmap",
326};
327
328/*
329 * Send subscribe request for one or more maps, according to
330 * monc->subs.
331 */
332static void __send_subscribe(struct ceph_mon_client *monc)
333{
334 struct ceph_msg *msg = monc->m_subscribe;
335 void *p = msg->front.iov_base;
336 void *const end = p + msg->front_alloc_len;
337 int num = 0;
338 int i;
339
340 dout("%s sent %lu\n", __func__, monc->sub_renew_sent);
341
342 BUG_ON(monc->cur_mon < 0);
343
344 if (!monc->sub_renew_sent)
345 monc->sub_renew_sent = jiffies | 1; /* never 0 */
346
347 msg->hdr.version = cpu_to_le16(2);
348
349 for (i = 0; i < ARRAY_SIZE(monc->subs); i++) {
350 if (monc->subs[i].want)
351 num++;
352 }
353 BUG_ON(num < 1); /* monmap sub is always there */
354 ceph_encode_32(&p, num);
355 for (i = 0; i < ARRAY_SIZE(monc->subs); i++) {
356 char buf[32];
357 int len;
358
359 if (!monc->subs[i].want)
360 continue;
361
362 len = sprintf(buf, "%s", ceph_sub_str[i]);
363 if (i == CEPH_SUB_MDSMAP &&
364 monc->fs_cluster_id != CEPH_FS_CLUSTER_ID_NONE)
365 len += sprintf(buf + len, ".%d", monc->fs_cluster_id);
366
367 dout("%s %s start %llu flags 0x%x\n", __func__, buf,
368 le64_to_cpu(monc->subs[i].item.start),
369 monc->subs[i].item.flags);
370 ceph_encode_string(&p, end, buf, len);
371 memcpy(p, &monc->subs[i].item, sizeof(monc->subs[i].item));
372 p += sizeof(monc->subs[i].item);
373 }
374
375 BUG_ON(p > end);
376 msg->front.iov_len = p - msg->front.iov_base;
377 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
378 ceph_msg_revoke(msg);
379 ceph_con_send(&monc->con, ceph_msg_get(msg));
380}
381
382static void handle_subscribe_ack(struct ceph_mon_client *monc,
383 struct ceph_msg *msg)
384{
385 unsigned int seconds;
386 struct ceph_mon_subscribe_ack *h = msg->front.iov_base;
387
388 if (msg->front.iov_len < sizeof(*h))
389 goto bad;
390 seconds = le32_to_cpu(h->duration);
391
392 mutex_lock(&monc->mutex);
393 if (monc->sub_renew_sent) {
394 /*
395 * This is only needed for legacy (infernalis or older)
396 * MONs -- see delayed_work().
397 */
398 monc->sub_renew_after = monc->sub_renew_sent +
399 (seconds >> 1) * HZ - 1;
400 dout("%s sent %lu duration %d renew after %lu\n", __func__,
401 monc->sub_renew_sent, seconds, monc->sub_renew_after);
402 monc->sub_renew_sent = 0;
403 } else {
404 dout("%s sent %lu renew after %lu, ignoring\n", __func__,
405 monc->sub_renew_sent, monc->sub_renew_after);
406 }
407 mutex_unlock(&monc->mutex);
408 return;
409bad:
410 pr_err("got corrupt subscribe-ack msg\n");
411 ceph_msg_dump(msg);
412}
413
414/*
415 * Register interest in a map
416 *
417 * @sub: one of CEPH_SUB_*
418 * @epoch: X for "every map since X", or 0 for "just the latest"
419 */
420static bool __ceph_monc_want_map(struct ceph_mon_client *monc, int sub,
421 u32 epoch, bool continuous)
422{
423 __le64 start = cpu_to_le64(epoch);
424 u8 flags = !continuous ? CEPH_SUBSCRIBE_ONETIME : 0;
425
426 dout("%s %s epoch %u continuous %d\n", __func__, ceph_sub_str[sub],
427 epoch, continuous);
428
429 if (monc->subs[sub].want &&
430 monc->subs[sub].item.start == start &&
431 monc->subs[sub].item.flags == flags)
432 return false;
433
434 monc->subs[sub].item.start = start;
435 monc->subs[sub].item.flags = flags;
436 monc->subs[sub].want = true;
437
438 return true;
439}
440
441bool ceph_monc_want_map(struct ceph_mon_client *monc, int sub, u32 epoch,
442 bool continuous)
443{
444 bool need_request;
445
446 mutex_lock(&monc->mutex);
447 need_request = __ceph_monc_want_map(monc, sub, epoch, continuous);
448 mutex_unlock(&monc->mutex);
449
450 return need_request;
451}
452EXPORT_SYMBOL(ceph_monc_want_map);
453
454/*
455 * Keep track of which maps we have
456 *
457 * @sub: one of CEPH_SUB_*
458 */
459static void __ceph_monc_got_map(struct ceph_mon_client *monc, int sub,
460 u32 epoch)
461{
462 dout("%s %s epoch %u\n", __func__, ceph_sub_str[sub], epoch);
463
464 if (monc->subs[sub].want) {
465 if (monc->subs[sub].item.flags & CEPH_SUBSCRIBE_ONETIME)
466 monc->subs[sub].want = false;
467 else
468 monc->subs[sub].item.start = cpu_to_le64(epoch + 1);
469 }
470
471 monc->subs[sub].have = epoch;
472}
473
474void ceph_monc_got_map(struct ceph_mon_client *monc, int sub, u32 epoch)
475{
476 mutex_lock(&monc->mutex);
477 __ceph_monc_got_map(monc, sub, epoch);
478 mutex_unlock(&monc->mutex);
479}
480EXPORT_SYMBOL(ceph_monc_got_map);
481
482void ceph_monc_renew_subs(struct ceph_mon_client *monc)
483{
484 mutex_lock(&monc->mutex);
485 __send_subscribe(monc);
486 mutex_unlock(&monc->mutex);
487}
488EXPORT_SYMBOL(ceph_monc_renew_subs);
489
490/*
491 * Wait for an osdmap with a given epoch.
492 *
493 * @epoch: epoch to wait for
494 * @timeout: in jiffies, 0 means "wait forever"
495 */
496int ceph_monc_wait_osdmap(struct ceph_mon_client *monc, u32 epoch,
497 unsigned long timeout)
498{
499 unsigned long started = jiffies;
500 long ret;
501
502 mutex_lock(&monc->mutex);
503 while (monc->subs[CEPH_SUB_OSDMAP].have < epoch) {
504 mutex_unlock(&monc->mutex);
505
506 if (timeout && time_after_eq(jiffies, started + timeout))
507 return -ETIMEDOUT;
508
509 ret = wait_event_interruptible_timeout(monc->client->auth_wq,
510 monc->subs[CEPH_SUB_OSDMAP].have >= epoch,
511 ceph_timeout_jiffies(timeout));
512 if (ret < 0)
513 return ret;
514
515 mutex_lock(&monc->mutex);
516 }
517
518 mutex_unlock(&monc->mutex);
519 return 0;
520}
521EXPORT_SYMBOL(ceph_monc_wait_osdmap);
522
523/*
524 * Open a session with a random monitor. Request monmap and osdmap,
525 * which are waited upon in __ceph_open_session().
526 */
527int ceph_monc_open_session(struct ceph_mon_client *monc)
528{
529 mutex_lock(&monc->mutex);
530 __ceph_monc_want_map(monc, CEPH_SUB_MONMAP, 0, true);
531 __ceph_monc_want_map(monc, CEPH_SUB_OSDMAP, 0, false);
532 __open_session(monc);
533 __schedule_delayed(monc);
534 mutex_unlock(&monc->mutex);
535 return 0;
536}
537EXPORT_SYMBOL(ceph_monc_open_session);
538
539static void ceph_monc_handle_map(struct ceph_mon_client *monc,
540 struct ceph_msg *msg)
541{
542 struct ceph_client *client = monc->client;
543 struct ceph_monmap *monmap;
544 void *p, *end;
545
546 mutex_lock(&monc->mutex);
547
548 dout("handle_monmap\n");
549 p = msg->front.iov_base;
550 end = p + msg->front.iov_len;
551
552 monmap = ceph_monmap_decode(&p, end, ceph_msgr2(client));
553 if (IS_ERR(monmap)) {
554 pr_err("problem decoding monmap, %d\n",
555 (int)PTR_ERR(monmap));
556 ceph_msg_dump(msg);
557 goto out;
558 }
559
560 if (ceph_check_fsid(client, &monmap->fsid) < 0) {
561 kfree(monmap);
562 goto out;
563 }
564
565 kfree(monc->monmap);
566 monc->monmap = monmap;
567
568 __ceph_monc_got_map(monc, CEPH_SUB_MONMAP, monc->monmap->epoch);
569 client->have_fsid = true;
570
571out:
572 mutex_unlock(&monc->mutex);
573 wake_up_all(&client->auth_wq);
574}
575
576/*
577 * generic requests (currently statfs, mon_get_version)
578 */
579DEFINE_RB_FUNCS(generic_request, struct ceph_mon_generic_request, tid, node)
580
581static void release_generic_request(struct kref *kref)
582{
583 struct ceph_mon_generic_request *req =
584 container_of(kref, struct ceph_mon_generic_request, kref);
585
586 dout("%s greq %p request %p reply %p\n", __func__, req, req->request,
587 req->reply);
588 WARN_ON(!RB_EMPTY_NODE(&req->node));
589
590 if (req->reply)
591 ceph_msg_put(req->reply);
592 if (req->request)
593 ceph_msg_put(req->request);
594
595 kfree(req);
596}
597
598static void put_generic_request(struct ceph_mon_generic_request *req)
599{
600 if (req)
601 kref_put(&req->kref, release_generic_request);
602}
603
604static void get_generic_request(struct ceph_mon_generic_request *req)
605{
606 kref_get(&req->kref);
607}
608
609static struct ceph_mon_generic_request *
610alloc_generic_request(struct ceph_mon_client *monc, gfp_t gfp)
611{
612 struct ceph_mon_generic_request *req;
613
614 req = kzalloc(sizeof(*req), gfp);
615 if (!req)
616 return NULL;
617
618 req->monc = monc;
619 kref_init(&req->kref);
620 RB_CLEAR_NODE(&req->node);
621 init_completion(&req->completion);
622
623 dout("%s greq %p\n", __func__, req);
624 return req;
625}
626
627static void register_generic_request(struct ceph_mon_generic_request *req)
628{
629 struct ceph_mon_client *monc = req->monc;
630
631 WARN_ON(req->tid);
632
633 get_generic_request(req);
634 req->tid = ++monc->last_tid;
635 insert_generic_request(&monc->generic_request_tree, req);
636}
637
638static void send_generic_request(struct ceph_mon_client *monc,
639 struct ceph_mon_generic_request *req)
640{
641 WARN_ON(!req->tid);
642
643 dout("%s greq %p tid %llu\n", __func__, req, req->tid);
644 req->request->hdr.tid = cpu_to_le64(req->tid);
645 ceph_con_send(&monc->con, ceph_msg_get(req->request));
646}
647
648static void __finish_generic_request(struct ceph_mon_generic_request *req)
649{
650 struct ceph_mon_client *monc = req->monc;
651
652 dout("%s greq %p tid %llu\n", __func__, req, req->tid);
653 erase_generic_request(&monc->generic_request_tree, req);
654
655 ceph_msg_revoke(req->request);
656 ceph_msg_revoke_incoming(req->reply);
657}
658
659static void finish_generic_request(struct ceph_mon_generic_request *req)
660{
661 __finish_generic_request(req);
662 put_generic_request(req);
663}
664
665static void complete_generic_request(struct ceph_mon_generic_request *req)
666{
667 if (req->complete_cb)
668 req->complete_cb(req);
669 else
670 complete_all(&req->completion);
671 put_generic_request(req);
672}
673
674static void cancel_generic_request(struct ceph_mon_generic_request *req)
675{
676 struct ceph_mon_client *monc = req->monc;
677 struct ceph_mon_generic_request *lookup_req;
678
679 dout("%s greq %p tid %llu\n", __func__, req, req->tid);
680
681 mutex_lock(&monc->mutex);
682 lookup_req = lookup_generic_request(&monc->generic_request_tree,
683 req->tid);
684 if (lookup_req) {
685 WARN_ON(lookup_req != req);
686 finish_generic_request(req);
687 }
688
689 mutex_unlock(&monc->mutex);
690}
691
692static int wait_generic_request(struct ceph_mon_generic_request *req)
693{
694 int ret;
695
696 dout("%s greq %p tid %llu\n", __func__, req, req->tid);
697 ret = wait_for_completion_interruptible(&req->completion);
698 if (ret)
699 cancel_generic_request(req);
700 else
701 ret = req->result; /* completed */
702
703 return ret;
704}
705
706static struct ceph_msg *get_generic_reply(struct ceph_connection *con,
707 struct ceph_msg_header *hdr,
708 int *skip)
709{
710 struct ceph_mon_client *monc = con->private;
711 struct ceph_mon_generic_request *req;
712 u64 tid = le64_to_cpu(hdr->tid);
713 struct ceph_msg *m;
714
715 mutex_lock(&monc->mutex);
716 req = lookup_generic_request(&monc->generic_request_tree, tid);
717 if (!req) {
718 dout("get_generic_reply %lld dne\n", tid);
719 *skip = 1;
720 m = NULL;
721 } else {
722 dout("get_generic_reply %lld got %p\n", tid, req->reply);
723 *skip = 0;
724 m = ceph_msg_get(req->reply);
725 /*
726 * we don't need to track the connection reading into
727 * this reply because we only have one open connection
728 * at a time, ever.
729 */
730 }
731 mutex_unlock(&monc->mutex);
732 return m;
733}
734
735/*
736 * statfs
737 */
738static void handle_statfs_reply(struct ceph_mon_client *monc,
739 struct ceph_msg *msg)
740{
741 struct ceph_mon_generic_request *req;
742 struct ceph_mon_statfs_reply *reply = msg->front.iov_base;
743 u64 tid = le64_to_cpu(msg->hdr.tid);
744
745 dout("%s msg %p tid %llu\n", __func__, msg, tid);
746
747 if (msg->front.iov_len != sizeof(*reply))
748 goto bad;
749
750 mutex_lock(&monc->mutex);
751 req = lookup_generic_request(&monc->generic_request_tree, tid);
752 if (!req) {
753 mutex_unlock(&monc->mutex);
754 return;
755 }
756
757 req->result = 0;
758 *req->u.st = reply->st; /* struct */
759 __finish_generic_request(req);
760 mutex_unlock(&monc->mutex);
761
762 complete_generic_request(req);
763 return;
764
765bad:
766 pr_err("corrupt statfs reply, tid %llu\n", tid);
767 ceph_msg_dump(msg);
768}
769
770/*
771 * Do a synchronous statfs().
772 */
773int ceph_monc_do_statfs(struct ceph_mon_client *monc, u64 data_pool,
774 struct ceph_statfs *buf)
775{
776 struct ceph_mon_generic_request *req;
777 struct ceph_mon_statfs *h;
778 int ret = -ENOMEM;
779
780 req = alloc_generic_request(monc, GFP_NOFS);
781 if (!req)
782 goto out;
783
784 req->request = ceph_msg_new(CEPH_MSG_STATFS, sizeof(*h), GFP_NOFS,
785 true);
786 if (!req->request)
787 goto out;
788
789 req->reply = ceph_msg_new(CEPH_MSG_STATFS_REPLY, 64, GFP_NOFS, true);
790 if (!req->reply)
791 goto out;
792
793 req->u.st = buf;
794 req->request->hdr.version = cpu_to_le16(2);
795
796 mutex_lock(&monc->mutex);
797 register_generic_request(req);
798 /* fill out request */
799 h = req->request->front.iov_base;
800 h->monhdr.have_version = 0;
801 h->monhdr.session_mon = cpu_to_le16(-1);
802 h->monhdr.session_mon_tid = 0;
803 h->fsid = monc->monmap->fsid;
804 h->contains_data_pool = (data_pool != CEPH_NOPOOL);
805 h->data_pool = cpu_to_le64(data_pool);
806 send_generic_request(monc, req);
807 mutex_unlock(&monc->mutex);
808
809 ret = wait_generic_request(req);
810out:
811 put_generic_request(req);
812 return ret;
813}
814EXPORT_SYMBOL(ceph_monc_do_statfs);
815
816static void handle_get_version_reply(struct ceph_mon_client *monc,
817 struct ceph_msg *msg)
818{
819 struct ceph_mon_generic_request *req;
820 u64 tid = le64_to_cpu(msg->hdr.tid);
821 void *p = msg->front.iov_base;
822 void *end = p + msg->front_alloc_len;
823 u64 handle;
824
825 dout("%s msg %p tid %llu\n", __func__, msg, tid);
826
827 ceph_decode_need(&p, end, 2*sizeof(u64), bad);
828 handle = ceph_decode_64(&p);
829 if (tid != 0 && tid != handle)
830 goto bad;
831
832 mutex_lock(&monc->mutex);
833 req = lookup_generic_request(&monc->generic_request_tree, handle);
834 if (!req) {
835 mutex_unlock(&monc->mutex);
836 return;
837 }
838
839 req->result = 0;
840 req->u.newest = ceph_decode_64(&p);
841 __finish_generic_request(req);
842 mutex_unlock(&monc->mutex);
843
844 complete_generic_request(req);
845 return;
846
847bad:
848 pr_err("corrupt mon_get_version reply, tid %llu\n", tid);
849 ceph_msg_dump(msg);
850}
851
852static struct ceph_mon_generic_request *
853__ceph_monc_get_version(struct ceph_mon_client *monc, const char *what,
854 ceph_monc_callback_t cb, u64 private_data)
855{
856 struct ceph_mon_generic_request *req;
857
858 req = alloc_generic_request(monc, GFP_NOIO);
859 if (!req)
860 goto err_put_req;
861
862 req->request = ceph_msg_new(CEPH_MSG_MON_GET_VERSION,
863 sizeof(u64) + sizeof(u32) + strlen(what),
864 GFP_NOIO, true);
865 if (!req->request)
866 goto err_put_req;
867
868 req->reply = ceph_msg_new(CEPH_MSG_MON_GET_VERSION_REPLY, 32, GFP_NOIO,
869 true);
870 if (!req->reply)
871 goto err_put_req;
872
873 req->complete_cb = cb;
874 req->private_data = private_data;
875
876 mutex_lock(&monc->mutex);
877 register_generic_request(req);
878 {
879 void *p = req->request->front.iov_base;
880 void *const end = p + req->request->front_alloc_len;
881
882 ceph_encode_64(&p, req->tid); /* handle */
883 ceph_encode_string(&p, end, what, strlen(what));
884 WARN_ON(p != end);
885 }
886 send_generic_request(monc, req);
887 mutex_unlock(&monc->mutex);
888
889 return req;
890
891err_put_req:
892 put_generic_request(req);
893 return ERR_PTR(-ENOMEM);
894}
895
896/*
897 * Send MMonGetVersion and wait for the reply.
898 *
899 * @what: one of "mdsmap", "osdmap" or "monmap"
900 */
901int ceph_monc_get_version(struct ceph_mon_client *monc, const char *what,
902 u64 *newest)
903{
904 struct ceph_mon_generic_request *req;
905 int ret;
906
907 req = __ceph_monc_get_version(monc, what, NULL, 0);
908 if (IS_ERR(req))
909 return PTR_ERR(req);
910
911 ret = wait_generic_request(req);
912 if (!ret)
913 *newest = req->u.newest;
914
915 put_generic_request(req);
916 return ret;
917}
918EXPORT_SYMBOL(ceph_monc_get_version);
919
920/*
921 * Send MMonGetVersion,
922 *
923 * @what: one of "mdsmap", "osdmap" or "monmap"
924 */
925int ceph_monc_get_version_async(struct ceph_mon_client *monc, const char *what,
926 ceph_monc_callback_t cb, u64 private_data)
927{
928 struct ceph_mon_generic_request *req;
929
930 req = __ceph_monc_get_version(monc, what, cb, private_data);
931 if (IS_ERR(req))
932 return PTR_ERR(req);
933
934 put_generic_request(req);
935 return 0;
936}
937EXPORT_SYMBOL(ceph_monc_get_version_async);
938
939static void handle_command_ack(struct ceph_mon_client *monc,
940 struct ceph_msg *msg)
941{
942 struct ceph_mon_generic_request *req;
943 void *p = msg->front.iov_base;
944 void *const end = p + msg->front_alloc_len;
945 u64 tid = le64_to_cpu(msg->hdr.tid);
946
947 dout("%s msg %p tid %llu\n", __func__, msg, tid);
948
949 ceph_decode_need(&p, end, sizeof(struct ceph_mon_request_header) +
950 sizeof(u32), bad);
951 p += sizeof(struct ceph_mon_request_header);
952
953 mutex_lock(&monc->mutex);
954 req = lookup_generic_request(&monc->generic_request_tree, tid);
955 if (!req) {
956 mutex_unlock(&monc->mutex);
957 return;
958 }
959
960 req->result = ceph_decode_32(&p);
961 __finish_generic_request(req);
962 mutex_unlock(&monc->mutex);
963
964 complete_generic_request(req);
965 return;
966
967bad:
968 pr_err("corrupt mon_command ack, tid %llu\n", tid);
969 ceph_msg_dump(msg);
970}
971
972static __printf(2, 0)
973int do_mon_command_vargs(struct ceph_mon_client *monc, const char *fmt,
974 va_list ap)
975{
976 struct ceph_mon_generic_request *req;
977 struct ceph_mon_command *h;
978 int ret = -ENOMEM;
979 int len;
980
981 req = alloc_generic_request(monc, GFP_NOIO);
982 if (!req)
983 goto out;
984
985 req->request = ceph_msg_new(CEPH_MSG_MON_COMMAND, 256, GFP_NOIO, true);
986 if (!req->request)
987 goto out;
988
989 req->reply = ceph_msg_new(CEPH_MSG_MON_COMMAND_ACK, 512, GFP_NOIO,
990 true);
991 if (!req->reply)
992 goto out;
993
994 mutex_lock(&monc->mutex);
995 register_generic_request(req);
996 h = req->request->front.iov_base;
997 h->monhdr.have_version = 0;
998 h->monhdr.session_mon = cpu_to_le16(-1);
999 h->monhdr.session_mon_tid = 0;
1000 h->fsid = monc->monmap->fsid;
1001 h->num_strs = cpu_to_le32(1);
1002 len = vsprintf(h->str, fmt, ap);
1003 h->str_len = cpu_to_le32(len);
1004 send_generic_request(monc, req);
1005 mutex_unlock(&monc->mutex);
1006
1007 ret = wait_generic_request(req);
1008out:
1009 put_generic_request(req);
1010 return ret;
1011}
1012
1013static __printf(2, 3)
1014int do_mon_command(struct ceph_mon_client *monc, const char *fmt, ...)
1015{
1016 va_list ap;
1017 int ret;
1018
1019 va_start(ap, fmt);
1020 ret = do_mon_command_vargs(monc, fmt, ap);
1021 va_end(ap);
1022 return ret;
1023}
1024
1025int ceph_monc_blocklist_add(struct ceph_mon_client *monc,
1026 struct ceph_entity_addr *client_addr)
1027{
1028 int ret;
1029
1030 ret = do_mon_command(monc,
1031 "{ \"prefix\": \"osd blocklist\", \
1032 \"blocklistop\": \"add\", \
1033 \"addr\": \"%pISpc/%u\" }",
1034 &client_addr->in_addr,
1035 le32_to_cpu(client_addr->nonce));
1036 if (ret == -EINVAL) {
1037 /*
1038 * The monitor returns EINVAL on an unrecognized command.
1039 * Try the legacy command -- it is exactly the same except
1040 * for the name.
1041 */
1042 ret = do_mon_command(monc,
1043 "{ \"prefix\": \"osd blacklist\", \
1044 \"blacklistop\": \"add\", \
1045 \"addr\": \"%pISpc/%u\" }",
1046 &client_addr->in_addr,
1047 le32_to_cpu(client_addr->nonce));
1048 }
1049 if (ret)
1050 return ret;
1051
1052 /*
1053 * Make sure we have the osdmap that includes the blocklist
1054 * entry. This is needed to ensure that the OSDs pick up the
1055 * new blocklist before processing any future requests from
1056 * this client.
1057 */
1058 return ceph_wait_for_latest_osdmap(monc->client, 0);
1059}
1060EXPORT_SYMBOL(ceph_monc_blocklist_add);
1061
1062/*
1063 * Resend pending generic requests.
1064 */
1065static void __resend_generic_request(struct ceph_mon_client *monc)
1066{
1067 struct ceph_mon_generic_request *req;
1068 struct rb_node *p;
1069
1070 for (p = rb_first(&monc->generic_request_tree); p; p = rb_next(p)) {
1071 req = rb_entry(p, struct ceph_mon_generic_request, node);
1072 ceph_msg_revoke(req->request);
1073 ceph_msg_revoke_incoming(req->reply);
1074 ceph_con_send(&monc->con, ceph_msg_get(req->request));
1075 }
1076}
1077
1078/*
1079 * Delayed work. If we haven't mounted yet, retry. Otherwise,
1080 * renew/retry subscription as needed (in case it is timing out, or we
1081 * got an ENOMEM). And keep the monitor connection alive.
1082 */
1083static void delayed_work(struct work_struct *work)
1084{
1085 struct ceph_mon_client *monc =
1086 container_of(work, struct ceph_mon_client, delayed_work.work);
1087
1088 dout("monc delayed_work\n");
1089 mutex_lock(&monc->mutex);
1090 if (monc->hunting) {
1091 dout("%s continuing hunt\n", __func__);
1092 reopen_session(monc);
1093 } else {
1094 int is_auth = ceph_auth_is_authenticated(monc->auth);
1095 if (ceph_con_keepalive_expired(&monc->con,
1096 CEPH_MONC_PING_TIMEOUT)) {
1097 dout("monc keepalive timeout\n");
1098 is_auth = 0;
1099 reopen_session(monc);
1100 }
1101
1102 if (!monc->hunting) {
1103 ceph_con_keepalive(&monc->con);
1104 __validate_auth(monc);
1105 un_backoff(monc);
1106 }
1107
1108 if (is_auth &&
1109 !(monc->con.peer_features & CEPH_FEATURE_MON_STATEFUL_SUB)) {
1110 unsigned long now = jiffies;
1111
1112 dout("%s renew subs? now %lu renew after %lu\n",
1113 __func__, now, monc->sub_renew_after);
1114 if (time_after_eq(now, monc->sub_renew_after))
1115 __send_subscribe(monc);
1116 }
1117 }
1118 __schedule_delayed(monc);
1119 mutex_unlock(&monc->mutex);
1120}
1121
1122/*
1123 * On startup, we build a temporary monmap populated with the IPs
1124 * provided by mount(2).
1125 */
1126static int build_initial_monmap(struct ceph_mon_client *monc)
1127{
1128 __le32 my_type = ceph_msgr2(monc->client) ?
1129 CEPH_ENTITY_ADDR_TYPE_MSGR2 : CEPH_ENTITY_ADDR_TYPE_LEGACY;
1130 struct ceph_options *opt = monc->client->options;
1131 int num_mon = opt->num_mon;
1132 int i;
1133
1134 /* build initial monmap */
1135 monc->monmap = kzalloc(struct_size(monc->monmap, mon_inst, num_mon),
1136 GFP_KERNEL);
1137 if (!monc->monmap)
1138 return -ENOMEM;
1139 monc->monmap->num_mon = num_mon;
1140
1141 for (i = 0; i < num_mon; i++) {
1142 struct ceph_entity_inst *inst = &monc->monmap->mon_inst[i];
1143
1144 memcpy(&inst->addr.in_addr, &opt->mon_addr[i].in_addr,
1145 sizeof(inst->addr.in_addr));
1146 inst->addr.type = my_type;
1147 inst->addr.nonce = 0;
1148 inst->name.type = CEPH_ENTITY_TYPE_MON;
1149 inst->name.num = cpu_to_le64(i);
1150 }
1151 return 0;
1152}
1153
1154int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl)
1155{
1156 int err;
1157
1158 dout("init\n");
1159 memset(monc, 0, sizeof(*monc));
1160 monc->client = cl;
1161 mutex_init(&monc->mutex);
1162
1163 err = build_initial_monmap(monc);
1164 if (err)
1165 goto out;
1166
1167 /* connection */
1168 /* authentication */
1169 monc->auth = ceph_auth_init(cl->options->name, cl->options->key,
1170 cl->options->con_modes);
1171 if (IS_ERR(monc->auth)) {
1172 err = PTR_ERR(monc->auth);
1173 goto out_monmap;
1174 }
1175 monc->auth->want_keys =
1176 CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |
1177 CEPH_ENTITY_TYPE_OSD | CEPH_ENTITY_TYPE_MDS;
1178
1179 /* msgs */
1180 err = -ENOMEM;
1181 monc->m_subscribe_ack = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE_ACK,
1182 sizeof(struct ceph_mon_subscribe_ack),
1183 GFP_KERNEL, true);
1184 if (!monc->m_subscribe_ack)
1185 goto out_auth;
1186
1187 monc->m_subscribe = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE, 128,
1188 GFP_KERNEL, true);
1189 if (!monc->m_subscribe)
1190 goto out_subscribe_ack;
1191
1192 monc->m_auth_reply = ceph_msg_new(CEPH_MSG_AUTH_REPLY, 4096,
1193 GFP_KERNEL, true);
1194 if (!monc->m_auth_reply)
1195 goto out_subscribe;
1196
1197 monc->m_auth = ceph_msg_new(CEPH_MSG_AUTH, 4096, GFP_KERNEL, true);
1198 monc->pending_auth = 0;
1199 if (!monc->m_auth)
1200 goto out_auth_reply;
1201
1202 ceph_con_init(&monc->con, monc, &mon_con_ops,
1203 &monc->client->msgr);
1204
1205 monc->cur_mon = -1;
1206 monc->had_a_connection = false;
1207 monc->hunt_mult = 1;
1208
1209 INIT_DELAYED_WORK(&monc->delayed_work, delayed_work);
1210 monc->generic_request_tree = RB_ROOT;
1211 monc->last_tid = 0;
1212
1213 monc->fs_cluster_id = CEPH_FS_CLUSTER_ID_NONE;
1214
1215 return 0;
1216
1217out_auth_reply:
1218 ceph_msg_put(monc->m_auth_reply);
1219out_subscribe:
1220 ceph_msg_put(monc->m_subscribe);
1221out_subscribe_ack:
1222 ceph_msg_put(monc->m_subscribe_ack);
1223out_auth:
1224 ceph_auth_destroy(monc->auth);
1225out_monmap:
1226 kfree(monc->monmap);
1227out:
1228 return err;
1229}
1230EXPORT_SYMBOL(ceph_monc_init);
1231
1232void ceph_monc_stop(struct ceph_mon_client *monc)
1233{
1234 dout("stop\n");
1235 cancel_delayed_work_sync(&monc->delayed_work);
1236
1237 mutex_lock(&monc->mutex);
1238 __close_session(monc);
1239 monc->cur_mon = -1;
1240 mutex_unlock(&monc->mutex);
1241
1242 /*
1243 * flush msgr queue before we destroy ourselves to ensure that:
1244 * - any work that references our embedded con is finished.
1245 * - any osd_client or other work that may reference an authorizer
1246 * finishes before we shut down the auth subsystem.
1247 */
1248 ceph_msgr_flush();
1249
1250 ceph_auth_destroy(monc->auth);
1251
1252 WARN_ON(!RB_EMPTY_ROOT(&monc->generic_request_tree));
1253
1254 ceph_msg_put(monc->m_auth);
1255 ceph_msg_put(monc->m_auth_reply);
1256 ceph_msg_put(monc->m_subscribe);
1257 ceph_msg_put(monc->m_subscribe_ack);
1258
1259 kfree(monc->monmap);
1260}
1261EXPORT_SYMBOL(ceph_monc_stop);
1262
1263static void finish_hunting(struct ceph_mon_client *monc)
1264{
1265 if (monc->hunting) {
1266 dout("%s found mon%d\n", __func__, monc->cur_mon);
1267 monc->hunting = false;
1268 monc->had_a_connection = true;
1269 un_backoff(monc);
1270 __schedule_delayed(monc);
1271 }
1272}
1273
1274static void finish_auth(struct ceph_mon_client *monc, int auth_err,
1275 bool was_authed)
1276{
1277 dout("%s auth_err %d was_authed %d\n", __func__, auth_err, was_authed);
1278 WARN_ON(auth_err > 0);
1279
1280 monc->pending_auth = 0;
1281 if (auth_err) {
1282 monc->client->auth_err = auth_err;
1283 wake_up_all(&monc->client->auth_wq);
1284 return;
1285 }
1286
1287 if (!was_authed && ceph_auth_is_authenticated(monc->auth)) {
1288 dout("%s authenticated, starting session global_id %llu\n",
1289 __func__, monc->auth->global_id);
1290
1291 monc->client->msgr.inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
1292 monc->client->msgr.inst.name.num =
1293 cpu_to_le64(monc->auth->global_id);
1294
1295 __send_subscribe(monc);
1296 __resend_generic_request(monc);
1297
1298 pr_info("mon%d %s session established\n", monc->cur_mon,
1299 ceph_pr_addr(&monc->con.peer_addr));
1300 }
1301}
1302
1303static void handle_auth_reply(struct ceph_mon_client *monc,
1304 struct ceph_msg *msg)
1305{
1306 bool was_authed;
1307 int ret;
1308
1309 mutex_lock(&monc->mutex);
1310 was_authed = ceph_auth_is_authenticated(monc->auth);
1311 ret = ceph_handle_auth_reply(monc->auth, msg->front.iov_base,
1312 msg->front.iov_len,
1313 monc->m_auth->front.iov_base,
1314 monc->m_auth->front_alloc_len);
1315 if (ret > 0) {
1316 __send_prepared_auth_request(monc, ret);
1317 } else {
1318 finish_auth(monc, ret, was_authed);
1319 finish_hunting(monc);
1320 }
1321 mutex_unlock(&monc->mutex);
1322}
1323
1324static int __validate_auth(struct ceph_mon_client *monc)
1325{
1326 int ret;
1327
1328 if (monc->pending_auth)
1329 return 0;
1330
1331 ret = ceph_build_auth(monc->auth, monc->m_auth->front.iov_base,
1332 monc->m_auth->front_alloc_len);
1333 if (ret <= 0)
1334 return ret; /* either an error, or no need to authenticate */
1335 __send_prepared_auth_request(monc, ret);
1336 return 0;
1337}
1338
1339int ceph_monc_validate_auth(struct ceph_mon_client *monc)
1340{
1341 int ret;
1342
1343 mutex_lock(&monc->mutex);
1344 ret = __validate_auth(monc);
1345 mutex_unlock(&monc->mutex);
1346 return ret;
1347}
1348EXPORT_SYMBOL(ceph_monc_validate_auth);
1349
1350static int mon_get_auth_request(struct ceph_connection *con,
1351 void *buf, int *buf_len,
1352 void **authorizer, int *authorizer_len)
1353{
1354 struct ceph_mon_client *monc = con->private;
1355 int ret;
1356
1357 mutex_lock(&monc->mutex);
1358 ret = ceph_auth_get_request(monc->auth, buf, *buf_len);
1359 mutex_unlock(&monc->mutex);
1360 if (ret < 0)
1361 return ret;
1362
1363 *buf_len = ret;
1364 *authorizer = NULL;
1365 *authorizer_len = 0;
1366 return 0;
1367}
1368
1369static int mon_handle_auth_reply_more(struct ceph_connection *con,
1370 void *reply, int reply_len,
1371 void *buf, int *buf_len,
1372 void **authorizer, int *authorizer_len)
1373{
1374 struct ceph_mon_client *monc = con->private;
1375 int ret;
1376
1377 mutex_lock(&monc->mutex);
1378 ret = ceph_auth_handle_reply_more(monc->auth, reply, reply_len,
1379 buf, *buf_len);
1380 mutex_unlock(&monc->mutex);
1381 if (ret < 0)
1382 return ret;
1383
1384 *buf_len = ret;
1385 *authorizer = NULL;
1386 *authorizer_len = 0;
1387 return 0;
1388}
1389
1390static int mon_handle_auth_done(struct ceph_connection *con,
1391 u64 global_id, void *reply, int reply_len,
1392 u8 *session_key, int *session_key_len,
1393 u8 *con_secret, int *con_secret_len)
1394{
1395 struct ceph_mon_client *monc = con->private;
1396 bool was_authed;
1397 int ret;
1398
1399 mutex_lock(&monc->mutex);
1400 WARN_ON(!monc->hunting);
1401 was_authed = ceph_auth_is_authenticated(monc->auth);
1402 ret = ceph_auth_handle_reply_done(monc->auth, global_id,
1403 reply, reply_len,
1404 session_key, session_key_len,
1405 con_secret, con_secret_len);
1406 finish_auth(monc, ret, was_authed);
1407 if (!ret)
1408 finish_hunting(monc);
1409 mutex_unlock(&monc->mutex);
1410 return 0;
1411}
1412
1413static int mon_handle_auth_bad_method(struct ceph_connection *con,
1414 int used_proto, int result,
1415 const int *allowed_protos, int proto_cnt,
1416 const int *allowed_modes, int mode_cnt)
1417{
1418 struct ceph_mon_client *monc = con->private;
1419 bool was_authed;
1420
1421 mutex_lock(&monc->mutex);
1422 WARN_ON(!monc->hunting);
1423 was_authed = ceph_auth_is_authenticated(monc->auth);
1424 ceph_auth_handle_bad_method(monc->auth, used_proto, result,
1425 allowed_protos, proto_cnt,
1426 allowed_modes, mode_cnt);
1427 finish_auth(monc, -EACCES, was_authed);
1428 mutex_unlock(&monc->mutex);
1429 return 0;
1430}
1431
1432/*
1433 * handle incoming message
1434 */
1435static void mon_dispatch(struct ceph_connection *con, struct ceph_msg *msg)
1436{
1437 struct ceph_mon_client *monc = con->private;
1438 int type = le16_to_cpu(msg->hdr.type);
1439
1440 switch (type) {
1441 case CEPH_MSG_AUTH_REPLY:
1442 handle_auth_reply(monc, msg);
1443 break;
1444
1445 case CEPH_MSG_MON_SUBSCRIBE_ACK:
1446 handle_subscribe_ack(monc, msg);
1447 break;
1448
1449 case CEPH_MSG_STATFS_REPLY:
1450 handle_statfs_reply(monc, msg);
1451 break;
1452
1453 case CEPH_MSG_MON_GET_VERSION_REPLY:
1454 handle_get_version_reply(monc, msg);
1455 break;
1456
1457 case CEPH_MSG_MON_COMMAND_ACK:
1458 handle_command_ack(monc, msg);
1459 break;
1460
1461 case CEPH_MSG_MON_MAP:
1462 ceph_monc_handle_map(monc, msg);
1463 break;
1464
1465 case CEPH_MSG_OSD_MAP:
1466 ceph_osdc_handle_map(&monc->client->osdc, msg);
1467 break;
1468
1469 default:
1470 /* can the chained handler handle it? */
1471 if (monc->client->extra_mon_dispatch &&
1472 monc->client->extra_mon_dispatch(monc->client, msg) == 0)
1473 break;
1474
1475 pr_err("received unknown message type %d %s\n", type,
1476 ceph_msg_type_name(type));
1477 }
1478 ceph_msg_put(msg);
1479}
1480
1481/*
1482 * Allocate memory for incoming message
1483 */
1484static struct ceph_msg *mon_alloc_msg(struct ceph_connection *con,
1485 struct ceph_msg_header *hdr,
1486 int *skip)
1487{
1488 struct ceph_mon_client *monc = con->private;
1489 int type = le16_to_cpu(hdr->type);
1490 int front_len = le32_to_cpu(hdr->front_len);
1491 struct ceph_msg *m = NULL;
1492
1493 *skip = 0;
1494
1495 switch (type) {
1496 case CEPH_MSG_MON_SUBSCRIBE_ACK:
1497 m = ceph_msg_get(monc->m_subscribe_ack);
1498 break;
1499 case CEPH_MSG_STATFS_REPLY:
1500 case CEPH_MSG_MON_COMMAND_ACK:
1501 return get_generic_reply(con, hdr, skip);
1502 case CEPH_MSG_AUTH_REPLY:
1503 m = ceph_msg_get(monc->m_auth_reply);
1504 break;
1505 case CEPH_MSG_MON_GET_VERSION_REPLY:
1506 if (le64_to_cpu(hdr->tid) != 0)
1507 return get_generic_reply(con, hdr, skip);
1508
1509 /*
1510 * Older OSDs don't set reply tid even if the original
1511 * request had a non-zero tid. Work around this weirdness
1512 * by allocating a new message.
1513 */
1514 fallthrough;
1515 case CEPH_MSG_MON_MAP:
1516 case CEPH_MSG_MDS_MAP:
1517 case CEPH_MSG_OSD_MAP:
1518 case CEPH_MSG_FS_MAP_USER:
1519 m = ceph_msg_new(type, front_len, GFP_NOFS, false);
1520 if (!m)
1521 return NULL; /* ENOMEM--return skip == 0 */
1522 break;
1523 }
1524
1525 if (!m) {
1526 pr_info("alloc_msg unknown type %d\n", type);
1527 *skip = 1;
1528 } else if (front_len > m->front_alloc_len) {
1529 pr_warn("mon_alloc_msg front %d > prealloc %d (%u#%llu)\n",
1530 front_len, m->front_alloc_len,
1531 (unsigned int)con->peer_name.type,
1532 le64_to_cpu(con->peer_name.num));
1533 ceph_msg_put(m);
1534 m = ceph_msg_new(type, front_len, GFP_NOFS, false);
1535 }
1536
1537 return m;
1538}
1539
1540/*
1541 * If the monitor connection resets, pick a new monitor and resubmit
1542 * any pending requests.
1543 */
1544static void mon_fault(struct ceph_connection *con)
1545{
1546 struct ceph_mon_client *monc = con->private;
1547
1548 mutex_lock(&monc->mutex);
1549 dout("%s mon%d\n", __func__, monc->cur_mon);
1550 if (monc->cur_mon >= 0) {
1551 if (!monc->hunting) {
1552 dout("%s hunting for new mon\n", __func__);
1553 reopen_session(monc);
1554 __schedule_delayed(monc);
1555 } else {
1556 dout("%s already hunting\n", __func__);
1557 }
1558 }
1559 mutex_unlock(&monc->mutex);
1560}
1561
1562/*
1563 * We can ignore refcounting on the connection struct, as all references
1564 * will come from the messenger workqueue, which is drained prior to
1565 * mon_client destruction.
1566 */
1567static struct ceph_connection *mon_get_con(struct ceph_connection *con)
1568{
1569 return con;
1570}
1571
1572static void mon_put_con(struct ceph_connection *con)
1573{
1574}
1575
1576static const struct ceph_connection_operations mon_con_ops = {
1577 .get = mon_get_con,
1578 .put = mon_put_con,
1579 .alloc_msg = mon_alloc_msg,
1580 .dispatch = mon_dispatch,
1581 .fault = mon_fault,
1582 .get_auth_request = mon_get_auth_request,
1583 .handle_auth_reply_more = mon_handle_auth_reply_more,
1584 .handle_auth_done = mon_handle_auth_done,
1585 .handle_auth_bad_method = mon_handle_auth_bad_method,
1586};
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/ceph/ceph_debug.h>
3
4#include <linux/module.h>
5#include <linux/types.h>
6#include <linux/slab.h>
7#include <linux/random.h>
8#include <linux/sched.h>
9
10#include <linux/ceph/ceph_features.h>
11#include <linux/ceph/mon_client.h>
12#include <linux/ceph/libceph.h>
13#include <linux/ceph/debugfs.h>
14#include <linux/ceph/decode.h>
15#include <linux/ceph/auth.h>
16
17/*
18 * Interact with Ceph monitor cluster. Handle requests for new map
19 * versions, and periodically resend as needed. Also implement
20 * statfs() and umount().
21 *
22 * A small cluster of Ceph "monitors" are responsible for managing critical
23 * cluster configuration and state information. An odd number (e.g., 3, 5)
24 * of cmon daemons use a modified version of the Paxos part-time parliament
25 * algorithm to manage the MDS map (mds cluster membership), OSD map, and
26 * list of clients who have mounted the file system.
27 *
28 * We maintain an open, active session with a monitor at all times in order to
29 * receive timely MDSMap updates. We periodically send a keepalive byte on the
30 * TCP socket to ensure we detect a failure. If the connection does break, we
31 * randomly hunt for a new monitor. Once the connection is reestablished, we
32 * resend any outstanding requests.
33 */
34
35static const struct ceph_connection_operations mon_con_ops;
36
37static int __validate_auth(struct ceph_mon_client *monc);
38
39/*
40 * Decode a monmap blob (e.g., during mount).
41 */
42static struct ceph_monmap *ceph_monmap_decode(void *p, void *end)
43{
44 struct ceph_monmap *m = NULL;
45 int i, err = -EINVAL;
46 struct ceph_fsid fsid;
47 u32 epoch, num_mon;
48 u32 len;
49
50 ceph_decode_32_safe(&p, end, len, bad);
51 ceph_decode_need(&p, end, len, bad);
52
53 dout("monmap_decode %p %p len %d (%d)\n", p, end, len, (int)(end-p));
54 p += sizeof(u16); /* skip version */
55
56 ceph_decode_need(&p, end, sizeof(fsid) + 2*sizeof(u32), bad);
57 ceph_decode_copy(&p, &fsid, sizeof(fsid));
58 epoch = ceph_decode_32(&p);
59
60 num_mon = ceph_decode_32(&p);
61
62 if (num_mon > CEPH_MAX_MON)
63 goto bad;
64 m = kmalloc(struct_size(m, mon_inst, num_mon), GFP_NOFS);
65 if (m == NULL)
66 return ERR_PTR(-ENOMEM);
67 m->fsid = fsid;
68 m->epoch = epoch;
69 m->num_mon = num_mon;
70 for (i = 0; i < num_mon; ++i) {
71 struct ceph_entity_inst *inst = &m->mon_inst[i];
72
73 /* copy name portion */
74 ceph_decode_copy_safe(&p, end, &inst->name,
75 sizeof(inst->name), bad);
76 err = ceph_decode_entity_addr(&p, end, &inst->addr);
77 if (err)
78 goto bad;
79 }
80 dout("monmap_decode epoch %d, num_mon %d\n", m->epoch,
81 m->num_mon);
82 for (i = 0; i < m->num_mon; i++)
83 dout("monmap_decode mon%d is %s\n", i,
84 ceph_pr_addr(&m->mon_inst[i].addr));
85 return m;
86bad:
87 dout("monmap_decode failed with %d\n", err);
88 kfree(m);
89 return ERR_PTR(err);
90}
91
92/*
93 * return true if *addr is included in the monmap.
94 */
95int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr)
96{
97 int i;
98
99 for (i = 0; i < m->num_mon; i++)
100 if (memcmp(addr, &m->mon_inst[i].addr, sizeof(*addr)) == 0)
101 return 1;
102 return 0;
103}
104
105/*
106 * Send an auth request.
107 */
108static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len)
109{
110 monc->pending_auth = 1;
111 monc->m_auth->front.iov_len = len;
112 monc->m_auth->hdr.front_len = cpu_to_le32(len);
113 ceph_msg_revoke(monc->m_auth);
114 ceph_msg_get(monc->m_auth); /* keep our ref */
115 ceph_con_send(&monc->con, monc->m_auth);
116}
117
118/*
119 * Close monitor session, if any.
120 */
121static void __close_session(struct ceph_mon_client *monc)
122{
123 dout("__close_session closing mon%d\n", monc->cur_mon);
124 ceph_msg_revoke(monc->m_auth);
125 ceph_msg_revoke_incoming(monc->m_auth_reply);
126 ceph_msg_revoke(monc->m_subscribe);
127 ceph_msg_revoke_incoming(monc->m_subscribe_ack);
128 ceph_con_close(&monc->con);
129
130 monc->pending_auth = 0;
131 ceph_auth_reset(monc->auth);
132}
133
134/*
135 * Pick a new monitor at random and set cur_mon. If we are repicking
136 * (i.e. cur_mon is already set), be sure to pick a different one.
137 */
138static void pick_new_mon(struct ceph_mon_client *monc)
139{
140 int old_mon = monc->cur_mon;
141
142 BUG_ON(monc->monmap->num_mon < 1);
143
144 if (monc->monmap->num_mon == 1) {
145 monc->cur_mon = 0;
146 } else {
147 int max = monc->monmap->num_mon;
148 int o = -1;
149 int n;
150
151 if (monc->cur_mon >= 0) {
152 if (monc->cur_mon < monc->monmap->num_mon)
153 o = monc->cur_mon;
154 if (o >= 0)
155 max--;
156 }
157
158 n = prandom_u32() % max;
159 if (o >= 0 && n >= o)
160 n++;
161
162 monc->cur_mon = n;
163 }
164
165 dout("%s mon%d -> mon%d out of %d mons\n", __func__, old_mon,
166 monc->cur_mon, monc->monmap->num_mon);
167}
168
169/*
170 * Open a session with a new monitor.
171 */
172static void __open_session(struct ceph_mon_client *monc)
173{
174 int ret;
175
176 pick_new_mon(monc);
177
178 monc->hunting = true;
179 if (monc->had_a_connection) {
180 monc->hunt_mult *= CEPH_MONC_HUNT_BACKOFF;
181 if (monc->hunt_mult > CEPH_MONC_HUNT_MAX_MULT)
182 monc->hunt_mult = CEPH_MONC_HUNT_MAX_MULT;
183 }
184
185 monc->sub_renew_after = jiffies; /* i.e., expired */
186 monc->sub_renew_sent = 0;
187
188 dout("%s opening mon%d\n", __func__, monc->cur_mon);
189 ceph_con_open(&monc->con, CEPH_ENTITY_TYPE_MON, monc->cur_mon,
190 &monc->monmap->mon_inst[monc->cur_mon].addr);
191
192 /*
193 * send an initial keepalive to ensure our timestamp is valid
194 * by the time we are in an OPENED state
195 */
196 ceph_con_keepalive(&monc->con);
197
198 /* initiate authentication handshake */
199 ret = ceph_auth_build_hello(monc->auth,
200 monc->m_auth->front.iov_base,
201 monc->m_auth->front_alloc_len);
202 BUG_ON(ret <= 0);
203 __send_prepared_auth_request(monc, ret);
204}
205
206static void reopen_session(struct ceph_mon_client *monc)
207{
208 if (!monc->hunting)
209 pr_info("mon%d %s session lost, hunting for new mon\n",
210 monc->cur_mon, ceph_pr_addr(&monc->con.peer_addr));
211
212 __close_session(monc);
213 __open_session(monc);
214}
215
216void ceph_monc_reopen_session(struct ceph_mon_client *monc)
217{
218 mutex_lock(&monc->mutex);
219 reopen_session(monc);
220 mutex_unlock(&monc->mutex);
221}
222
223static void un_backoff(struct ceph_mon_client *monc)
224{
225 monc->hunt_mult /= 2; /* reduce by 50% */
226 if (monc->hunt_mult < 1)
227 monc->hunt_mult = 1;
228 dout("%s hunt_mult now %d\n", __func__, monc->hunt_mult);
229}
230
231/*
232 * Reschedule delayed work timer.
233 */
234static void __schedule_delayed(struct ceph_mon_client *monc)
235{
236 unsigned long delay;
237
238 if (monc->hunting)
239 delay = CEPH_MONC_HUNT_INTERVAL * monc->hunt_mult;
240 else
241 delay = CEPH_MONC_PING_INTERVAL;
242
243 dout("__schedule_delayed after %lu\n", delay);
244 mod_delayed_work(system_wq, &monc->delayed_work,
245 round_jiffies_relative(delay));
246}
247
248const char *ceph_sub_str[] = {
249 [CEPH_SUB_MONMAP] = "monmap",
250 [CEPH_SUB_OSDMAP] = "osdmap",
251 [CEPH_SUB_FSMAP] = "fsmap.user",
252 [CEPH_SUB_MDSMAP] = "mdsmap",
253};
254
255/*
256 * Send subscribe request for one or more maps, according to
257 * monc->subs.
258 */
259static void __send_subscribe(struct ceph_mon_client *monc)
260{
261 struct ceph_msg *msg = monc->m_subscribe;
262 void *p = msg->front.iov_base;
263 void *const end = p + msg->front_alloc_len;
264 int num = 0;
265 int i;
266
267 dout("%s sent %lu\n", __func__, monc->sub_renew_sent);
268
269 BUG_ON(monc->cur_mon < 0);
270
271 if (!monc->sub_renew_sent)
272 monc->sub_renew_sent = jiffies | 1; /* never 0 */
273
274 msg->hdr.version = cpu_to_le16(2);
275
276 for (i = 0; i < ARRAY_SIZE(monc->subs); i++) {
277 if (monc->subs[i].want)
278 num++;
279 }
280 BUG_ON(num < 1); /* monmap sub is always there */
281 ceph_encode_32(&p, num);
282 for (i = 0; i < ARRAY_SIZE(monc->subs); i++) {
283 char buf[32];
284 int len;
285
286 if (!monc->subs[i].want)
287 continue;
288
289 len = sprintf(buf, "%s", ceph_sub_str[i]);
290 if (i == CEPH_SUB_MDSMAP &&
291 monc->fs_cluster_id != CEPH_FS_CLUSTER_ID_NONE)
292 len += sprintf(buf + len, ".%d", monc->fs_cluster_id);
293
294 dout("%s %s start %llu flags 0x%x\n", __func__, buf,
295 le64_to_cpu(monc->subs[i].item.start),
296 monc->subs[i].item.flags);
297 ceph_encode_string(&p, end, buf, len);
298 memcpy(p, &monc->subs[i].item, sizeof(monc->subs[i].item));
299 p += sizeof(monc->subs[i].item);
300 }
301
302 BUG_ON(p > end);
303 msg->front.iov_len = p - msg->front.iov_base;
304 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
305 ceph_msg_revoke(msg);
306 ceph_con_send(&monc->con, ceph_msg_get(msg));
307}
308
309static void handle_subscribe_ack(struct ceph_mon_client *monc,
310 struct ceph_msg *msg)
311{
312 unsigned int seconds;
313 struct ceph_mon_subscribe_ack *h = msg->front.iov_base;
314
315 if (msg->front.iov_len < sizeof(*h))
316 goto bad;
317 seconds = le32_to_cpu(h->duration);
318
319 mutex_lock(&monc->mutex);
320 if (monc->sub_renew_sent) {
321 /*
322 * This is only needed for legacy (infernalis or older)
323 * MONs -- see delayed_work().
324 */
325 monc->sub_renew_after = monc->sub_renew_sent +
326 (seconds >> 1) * HZ - 1;
327 dout("%s sent %lu duration %d renew after %lu\n", __func__,
328 monc->sub_renew_sent, seconds, monc->sub_renew_after);
329 monc->sub_renew_sent = 0;
330 } else {
331 dout("%s sent %lu renew after %lu, ignoring\n", __func__,
332 monc->sub_renew_sent, monc->sub_renew_after);
333 }
334 mutex_unlock(&monc->mutex);
335 return;
336bad:
337 pr_err("got corrupt subscribe-ack msg\n");
338 ceph_msg_dump(msg);
339}
340
341/*
342 * Register interest in a map
343 *
344 * @sub: one of CEPH_SUB_*
345 * @epoch: X for "every map since X", or 0 for "just the latest"
346 */
347static bool __ceph_monc_want_map(struct ceph_mon_client *monc, int sub,
348 u32 epoch, bool continuous)
349{
350 __le64 start = cpu_to_le64(epoch);
351 u8 flags = !continuous ? CEPH_SUBSCRIBE_ONETIME : 0;
352
353 dout("%s %s epoch %u continuous %d\n", __func__, ceph_sub_str[sub],
354 epoch, continuous);
355
356 if (monc->subs[sub].want &&
357 monc->subs[sub].item.start == start &&
358 monc->subs[sub].item.flags == flags)
359 return false;
360
361 monc->subs[sub].item.start = start;
362 monc->subs[sub].item.flags = flags;
363 monc->subs[sub].want = true;
364
365 return true;
366}
367
368bool ceph_monc_want_map(struct ceph_mon_client *monc, int sub, u32 epoch,
369 bool continuous)
370{
371 bool need_request;
372
373 mutex_lock(&monc->mutex);
374 need_request = __ceph_monc_want_map(monc, sub, epoch, continuous);
375 mutex_unlock(&monc->mutex);
376
377 return need_request;
378}
379EXPORT_SYMBOL(ceph_monc_want_map);
380
381/*
382 * Keep track of which maps we have
383 *
384 * @sub: one of CEPH_SUB_*
385 */
386static void __ceph_monc_got_map(struct ceph_mon_client *monc, int sub,
387 u32 epoch)
388{
389 dout("%s %s epoch %u\n", __func__, ceph_sub_str[sub], epoch);
390
391 if (monc->subs[sub].want) {
392 if (monc->subs[sub].item.flags & CEPH_SUBSCRIBE_ONETIME)
393 monc->subs[sub].want = false;
394 else
395 monc->subs[sub].item.start = cpu_to_le64(epoch + 1);
396 }
397
398 monc->subs[sub].have = epoch;
399}
400
401void ceph_monc_got_map(struct ceph_mon_client *monc, int sub, u32 epoch)
402{
403 mutex_lock(&monc->mutex);
404 __ceph_monc_got_map(monc, sub, epoch);
405 mutex_unlock(&monc->mutex);
406}
407EXPORT_SYMBOL(ceph_monc_got_map);
408
409void ceph_monc_renew_subs(struct ceph_mon_client *monc)
410{
411 mutex_lock(&monc->mutex);
412 __send_subscribe(monc);
413 mutex_unlock(&monc->mutex);
414}
415EXPORT_SYMBOL(ceph_monc_renew_subs);
416
417/*
418 * Wait for an osdmap with a given epoch.
419 *
420 * @epoch: epoch to wait for
421 * @timeout: in jiffies, 0 means "wait forever"
422 */
423int ceph_monc_wait_osdmap(struct ceph_mon_client *monc, u32 epoch,
424 unsigned long timeout)
425{
426 unsigned long started = jiffies;
427 long ret;
428
429 mutex_lock(&monc->mutex);
430 while (monc->subs[CEPH_SUB_OSDMAP].have < epoch) {
431 mutex_unlock(&monc->mutex);
432
433 if (timeout && time_after_eq(jiffies, started + timeout))
434 return -ETIMEDOUT;
435
436 ret = wait_event_interruptible_timeout(monc->client->auth_wq,
437 monc->subs[CEPH_SUB_OSDMAP].have >= epoch,
438 ceph_timeout_jiffies(timeout));
439 if (ret < 0)
440 return ret;
441
442 mutex_lock(&monc->mutex);
443 }
444
445 mutex_unlock(&monc->mutex);
446 return 0;
447}
448EXPORT_SYMBOL(ceph_monc_wait_osdmap);
449
450/*
451 * Open a session with a random monitor. Request monmap and osdmap,
452 * which are waited upon in __ceph_open_session().
453 */
454int ceph_monc_open_session(struct ceph_mon_client *monc)
455{
456 mutex_lock(&monc->mutex);
457 __ceph_monc_want_map(monc, CEPH_SUB_MONMAP, 0, true);
458 __ceph_monc_want_map(monc, CEPH_SUB_OSDMAP, 0, false);
459 __open_session(monc);
460 __schedule_delayed(monc);
461 mutex_unlock(&monc->mutex);
462 return 0;
463}
464EXPORT_SYMBOL(ceph_monc_open_session);
465
466static void ceph_monc_handle_map(struct ceph_mon_client *monc,
467 struct ceph_msg *msg)
468{
469 struct ceph_client *client = monc->client;
470 struct ceph_monmap *monmap = NULL, *old = monc->monmap;
471 void *p, *end;
472
473 mutex_lock(&monc->mutex);
474
475 dout("handle_monmap\n");
476 p = msg->front.iov_base;
477 end = p + msg->front.iov_len;
478
479 monmap = ceph_monmap_decode(p, end);
480 if (IS_ERR(monmap)) {
481 pr_err("problem decoding monmap, %d\n",
482 (int)PTR_ERR(monmap));
483 ceph_msg_dump(msg);
484 goto out;
485 }
486
487 if (ceph_check_fsid(monc->client, &monmap->fsid) < 0) {
488 kfree(monmap);
489 goto out;
490 }
491
492 client->monc.monmap = monmap;
493 kfree(old);
494
495 __ceph_monc_got_map(monc, CEPH_SUB_MONMAP, monc->monmap->epoch);
496 client->have_fsid = true;
497
498out:
499 mutex_unlock(&monc->mutex);
500 wake_up_all(&client->auth_wq);
501}
502
503/*
504 * generic requests (currently statfs, mon_get_version)
505 */
506DEFINE_RB_FUNCS(generic_request, struct ceph_mon_generic_request, tid, node)
507
508static void release_generic_request(struct kref *kref)
509{
510 struct ceph_mon_generic_request *req =
511 container_of(kref, struct ceph_mon_generic_request, kref);
512
513 dout("%s greq %p request %p reply %p\n", __func__, req, req->request,
514 req->reply);
515 WARN_ON(!RB_EMPTY_NODE(&req->node));
516
517 if (req->reply)
518 ceph_msg_put(req->reply);
519 if (req->request)
520 ceph_msg_put(req->request);
521
522 kfree(req);
523}
524
525static void put_generic_request(struct ceph_mon_generic_request *req)
526{
527 if (req)
528 kref_put(&req->kref, release_generic_request);
529}
530
531static void get_generic_request(struct ceph_mon_generic_request *req)
532{
533 kref_get(&req->kref);
534}
535
536static struct ceph_mon_generic_request *
537alloc_generic_request(struct ceph_mon_client *monc, gfp_t gfp)
538{
539 struct ceph_mon_generic_request *req;
540
541 req = kzalloc(sizeof(*req), gfp);
542 if (!req)
543 return NULL;
544
545 req->monc = monc;
546 kref_init(&req->kref);
547 RB_CLEAR_NODE(&req->node);
548 init_completion(&req->completion);
549
550 dout("%s greq %p\n", __func__, req);
551 return req;
552}
553
554static void register_generic_request(struct ceph_mon_generic_request *req)
555{
556 struct ceph_mon_client *monc = req->monc;
557
558 WARN_ON(req->tid);
559
560 get_generic_request(req);
561 req->tid = ++monc->last_tid;
562 insert_generic_request(&monc->generic_request_tree, req);
563}
564
565static void send_generic_request(struct ceph_mon_client *monc,
566 struct ceph_mon_generic_request *req)
567{
568 WARN_ON(!req->tid);
569
570 dout("%s greq %p tid %llu\n", __func__, req, req->tid);
571 req->request->hdr.tid = cpu_to_le64(req->tid);
572 ceph_con_send(&monc->con, ceph_msg_get(req->request));
573}
574
575static void __finish_generic_request(struct ceph_mon_generic_request *req)
576{
577 struct ceph_mon_client *monc = req->monc;
578
579 dout("%s greq %p tid %llu\n", __func__, req, req->tid);
580 erase_generic_request(&monc->generic_request_tree, req);
581
582 ceph_msg_revoke(req->request);
583 ceph_msg_revoke_incoming(req->reply);
584}
585
586static void finish_generic_request(struct ceph_mon_generic_request *req)
587{
588 __finish_generic_request(req);
589 put_generic_request(req);
590}
591
592static void complete_generic_request(struct ceph_mon_generic_request *req)
593{
594 if (req->complete_cb)
595 req->complete_cb(req);
596 else
597 complete_all(&req->completion);
598 put_generic_request(req);
599}
600
601static void cancel_generic_request(struct ceph_mon_generic_request *req)
602{
603 struct ceph_mon_client *monc = req->monc;
604 struct ceph_mon_generic_request *lookup_req;
605
606 dout("%s greq %p tid %llu\n", __func__, req, req->tid);
607
608 mutex_lock(&monc->mutex);
609 lookup_req = lookup_generic_request(&monc->generic_request_tree,
610 req->tid);
611 if (lookup_req) {
612 WARN_ON(lookup_req != req);
613 finish_generic_request(req);
614 }
615
616 mutex_unlock(&monc->mutex);
617}
618
619static int wait_generic_request(struct ceph_mon_generic_request *req)
620{
621 int ret;
622
623 dout("%s greq %p tid %llu\n", __func__, req, req->tid);
624 ret = wait_for_completion_interruptible(&req->completion);
625 if (ret)
626 cancel_generic_request(req);
627 else
628 ret = req->result; /* completed */
629
630 return ret;
631}
632
633static struct ceph_msg *get_generic_reply(struct ceph_connection *con,
634 struct ceph_msg_header *hdr,
635 int *skip)
636{
637 struct ceph_mon_client *monc = con->private;
638 struct ceph_mon_generic_request *req;
639 u64 tid = le64_to_cpu(hdr->tid);
640 struct ceph_msg *m;
641
642 mutex_lock(&monc->mutex);
643 req = lookup_generic_request(&monc->generic_request_tree, tid);
644 if (!req) {
645 dout("get_generic_reply %lld dne\n", tid);
646 *skip = 1;
647 m = NULL;
648 } else {
649 dout("get_generic_reply %lld got %p\n", tid, req->reply);
650 *skip = 0;
651 m = ceph_msg_get(req->reply);
652 /*
653 * we don't need to track the connection reading into
654 * this reply because we only have one open connection
655 * at a time, ever.
656 */
657 }
658 mutex_unlock(&monc->mutex);
659 return m;
660}
661
662/*
663 * statfs
664 */
665static void handle_statfs_reply(struct ceph_mon_client *monc,
666 struct ceph_msg *msg)
667{
668 struct ceph_mon_generic_request *req;
669 struct ceph_mon_statfs_reply *reply = msg->front.iov_base;
670 u64 tid = le64_to_cpu(msg->hdr.tid);
671
672 dout("%s msg %p tid %llu\n", __func__, msg, tid);
673
674 if (msg->front.iov_len != sizeof(*reply))
675 goto bad;
676
677 mutex_lock(&monc->mutex);
678 req = lookup_generic_request(&monc->generic_request_tree, tid);
679 if (!req) {
680 mutex_unlock(&monc->mutex);
681 return;
682 }
683
684 req->result = 0;
685 *req->u.st = reply->st; /* struct */
686 __finish_generic_request(req);
687 mutex_unlock(&monc->mutex);
688
689 complete_generic_request(req);
690 return;
691
692bad:
693 pr_err("corrupt statfs reply, tid %llu\n", tid);
694 ceph_msg_dump(msg);
695}
696
697/*
698 * Do a synchronous statfs().
699 */
700int ceph_monc_do_statfs(struct ceph_mon_client *monc, u64 data_pool,
701 struct ceph_statfs *buf)
702{
703 struct ceph_mon_generic_request *req;
704 struct ceph_mon_statfs *h;
705 int ret = -ENOMEM;
706
707 req = alloc_generic_request(monc, GFP_NOFS);
708 if (!req)
709 goto out;
710
711 req->request = ceph_msg_new(CEPH_MSG_STATFS, sizeof(*h), GFP_NOFS,
712 true);
713 if (!req->request)
714 goto out;
715
716 req->reply = ceph_msg_new(CEPH_MSG_STATFS_REPLY, 64, GFP_NOFS, true);
717 if (!req->reply)
718 goto out;
719
720 req->u.st = buf;
721 req->request->hdr.version = cpu_to_le16(2);
722
723 mutex_lock(&monc->mutex);
724 register_generic_request(req);
725 /* fill out request */
726 h = req->request->front.iov_base;
727 h->monhdr.have_version = 0;
728 h->monhdr.session_mon = cpu_to_le16(-1);
729 h->monhdr.session_mon_tid = 0;
730 h->fsid = monc->monmap->fsid;
731 h->contains_data_pool = (data_pool != CEPH_NOPOOL);
732 h->data_pool = cpu_to_le64(data_pool);
733 send_generic_request(monc, req);
734 mutex_unlock(&monc->mutex);
735
736 ret = wait_generic_request(req);
737out:
738 put_generic_request(req);
739 return ret;
740}
741EXPORT_SYMBOL(ceph_monc_do_statfs);
742
743static void handle_get_version_reply(struct ceph_mon_client *monc,
744 struct ceph_msg *msg)
745{
746 struct ceph_mon_generic_request *req;
747 u64 tid = le64_to_cpu(msg->hdr.tid);
748 void *p = msg->front.iov_base;
749 void *end = p + msg->front_alloc_len;
750 u64 handle;
751
752 dout("%s msg %p tid %llu\n", __func__, msg, tid);
753
754 ceph_decode_need(&p, end, 2*sizeof(u64), bad);
755 handle = ceph_decode_64(&p);
756 if (tid != 0 && tid != handle)
757 goto bad;
758
759 mutex_lock(&monc->mutex);
760 req = lookup_generic_request(&monc->generic_request_tree, handle);
761 if (!req) {
762 mutex_unlock(&monc->mutex);
763 return;
764 }
765
766 req->result = 0;
767 req->u.newest = ceph_decode_64(&p);
768 __finish_generic_request(req);
769 mutex_unlock(&monc->mutex);
770
771 complete_generic_request(req);
772 return;
773
774bad:
775 pr_err("corrupt mon_get_version reply, tid %llu\n", tid);
776 ceph_msg_dump(msg);
777}
778
779static struct ceph_mon_generic_request *
780__ceph_monc_get_version(struct ceph_mon_client *monc, const char *what,
781 ceph_monc_callback_t cb, u64 private_data)
782{
783 struct ceph_mon_generic_request *req;
784
785 req = alloc_generic_request(monc, GFP_NOIO);
786 if (!req)
787 goto err_put_req;
788
789 req->request = ceph_msg_new(CEPH_MSG_MON_GET_VERSION,
790 sizeof(u64) + sizeof(u32) + strlen(what),
791 GFP_NOIO, true);
792 if (!req->request)
793 goto err_put_req;
794
795 req->reply = ceph_msg_new(CEPH_MSG_MON_GET_VERSION_REPLY, 32, GFP_NOIO,
796 true);
797 if (!req->reply)
798 goto err_put_req;
799
800 req->complete_cb = cb;
801 req->private_data = private_data;
802
803 mutex_lock(&monc->mutex);
804 register_generic_request(req);
805 {
806 void *p = req->request->front.iov_base;
807 void *const end = p + req->request->front_alloc_len;
808
809 ceph_encode_64(&p, req->tid); /* handle */
810 ceph_encode_string(&p, end, what, strlen(what));
811 WARN_ON(p != end);
812 }
813 send_generic_request(monc, req);
814 mutex_unlock(&monc->mutex);
815
816 return req;
817
818err_put_req:
819 put_generic_request(req);
820 return ERR_PTR(-ENOMEM);
821}
822
823/*
824 * Send MMonGetVersion and wait for the reply.
825 *
826 * @what: one of "mdsmap", "osdmap" or "monmap"
827 */
828int ceph_monc_get_version(struct ceph_mon_client *monc, const char *what,
829 u64 *newest)
830{
831 struct ceph_mon_generic_request *req;
832 int ret;
833
834 req = __ceph_monc_get_version(monc, what, NULL, 0);
835 if (IS_ERR(req))
836 return PTR_ERR(req);
837
838 ret = wait_generic_request(req);
839 if (!ret)
840 *newest = req->u.newest;
841
842 put_generic_request(req);
843 return ret;
844}
845EXPORT_SYMBOL(ceph_monc_get_version);
846
847/*
848 * Send MMonGetVersion,
849 *
850 * @what: one of "mdsmap", "osdmap" or "monmap"
851 */
852int ceph_monc_get_version_async(struct ceph_mon_client *monc, const char *what,
853 ceph_monc_callback_t cb, u64 private_data)
854{
855 struct ceph_mon_generic_request *req;
856
857 req = __ceph_monc_get_version(monc, what, cb, private_data);
858 if (IS_ERR(req))
859 return PTR_ERR(req);
860
861 put_generic_request(req);
862 return 0;
863}
864EXPORT_SYMBOL(ceph_monc_get_version_async);
865
866static void handle_command_ack(struct ceph_mon_client *monc,
867 struct ceph_msg *msg)
868{
869 struct ceph_mon_generic_request *req;
870 void *p = msg->front.iov_base;
871 void *const end = p + msg->front_alloc_len;
872 u64 tid = le64_to_cpu(msg->hdr.tid);
873
874 dout("%s msg %p tid %llu\n", __func__, msg, tid);
875
876 ceph_decode_need(&p, end, sizeof(struct ceph_mon_request_header) +
877 sizeof(u32), bad);
878 p += sizeof(struct ceph_mon_request_header);
879
880 mutex_lock(&monc->mutex);
881 req = lookup_generic_request(&monc->generic_request_tree, tid);
882 if (!req) {
883 mutex_unlock(&monc->mutex);
884 return;
885 }
886
887 req->result = ceph_decode_32(&p);
888 __finish_generic_request(req);
889 mutex_unlock(&monc->mutex);
890
891 complete_generic_request(req);
892 return;
893
894bad:
895 pr_err("corrupt mon_command ack, tid %llu\n", tid);
896 ceph_msg_dump(msg);
897}
898
899int ceph_monc_blacklist_add(struct ceph_mon_client *monc,
900 struct ceph_entity_addr *client_addr)
901{
902 struct ceph_mon_generic_request *req;
903 struct ceph_mon_command *h;
904 int ret = -ENOMEM;
905 int len;
906
907 req = alloc_generic_request(monc, GFP_NOIO);
908 if (!req)
909 goto out;
910
911 req->request = ceph_msg_new(CEPH_MSG_MON_COMMAND, 256, GFP_NOIO, true);
912 if (!req->request)
913 goto out;
914
915 req->reply = ceph_msg_new(CEPH_MSG_MON_COMMAND_ACK, 512, GFP_NOIO,
916 true);
917 if (!req->reply)
918 goto out;
919
920 mutex_lock(&monc->mutex);
921 register_generic_request(req);
922 h = req->request->front.iov_base;
923 h->monhdr.have_version = 0;
924 h->monhdr.session_mon = cpu_to_le16(-1);
925 h->monhdr.session_mon_tid = 0;
926 h->fsid = monc->monmap->fsid;
927 h->num_strs = cpu_to_le32(1);
928 len = sprintf(h->str, "{ \"prefix\": \"osd blacklist\", \
929 \"blacklistop\": \"add\", \
930 \"addr\": \"%pISpc/%u\" }",
931 &client_addr->in_addr, le32_to_cpu(client_addr->nonce));
932 h->str_len = cpu_to_le32(len);
933 send_generic_request(monc, req);
934 mutex_unlock(&monc->mutex);
935
936 ret = wait_generic_request(req);
937 if (!ret)
938 /*
939 * Make sure we have the osdmap that includes the blacklist
940 * entry. This is needed to ensure that the OSDs pick up the
941 * new blacklist before processing any future requests from
942 * this client.
943 */
944 ret = ceph_wait_for_latest_osdmap(monc->client, 0);
945
946out:
947 put_generic_request(req);
948 return ret;
949}
950EXPORT_SYMBOL(ceph_monc_blacklist_add);
951
952/*
953 * Resend pending generic requests.
954 */
955static void __resend_generic_request(struct ceph_mon_client *monc)
956{
957 struct ceph_mon_generic_request *req;
958 struct rb_node *p;
959
960 for (p = rb_first(&monc->generic_request_tree); p; p = rb_next(p)) {
961 req = rb_entry(p, struct ceph_mon_generic_request, node);
962 ceph_msg_revoke(req->request);
963 ceph_msg_revoke_incoming(req->reply);
964 ceph_con_send(&monc->con, ceph_msg_get(req->request));
965 }
966}
967
968/*
969 * Delayed work. If we haven't mounted yet, retry. Otherwise,
970 * renew/retry subscription as needed (in case it is timing out, or we
971 * got an ENOMEM). And keep the monitor connection alive.
972 */
973static void delayed_work(struct work_struct *work)
974{
975 struct ceph_mon_client *monc =
976 container_of(work, struct ceph_mon_client, delayed_work.work);
977
978 dout("monc delayed_work\n");
979 mutex_lock(&monc->mutex);
980 if (monc->hunting) {
981 dout("%s continuing hunt\n", __func__);
982 reopen_session(monc);
983 } else {
984 int is_auth = ceph_auth_is_authenticated(monc->auth);
985 if (ceph_con_keepalive_expired(&monc->con,
986 CEPH_MONC_PING_TIMEOUT)) {
987 dout("monc keepalive timeout\n");
988 is_auth = 0;
989 reopen_session(monc);
990 }
991
992 if (!monc->hunting) {
993 ceph_con_keepalive(&monc->con);
994 __validate_auth(monc);
995 un_backoff(monc);
996 }
997
998 if (is_auth &&
999 !(monc->con.peer_features & CEPH_FEATURE_MON_STATEFUL_SUB)) {
1000 unsigned long now = jiffies;
1001
1002 dout("%s renew subs? now %lu renew after %lu\n",
1003 __func__, now, monc->sub_renew_after);
1004 if (time_after_eq(now, monc->sub_renew_after))
1005 __send_subscribe(monc);
1006 }
1007 }
1008 __schedule_delayed(monc);
1009 mutex_unlock(&monc->mutex);
1010}
1011
1012/*
1013 * On startup, we build a temporary monmap populated with the IPs
1014 * provided by mount(2).
1015 */
1016static int build_initial_monmap(struct ceph_mon_client *monc)
1017{
1018 struct ceph_options *opt = monc->client->options;
1019 struct ceph_entity_addr *mon_addr = opt->mon_addr;
1020 int num_mon = opt->num_mon;
1021 int i;
1022
1023 /* build initial monmap */
1024 monc->monmap = kzalloc(struct_size(monc->monmap, mon_inst, num_mon),
1025 GFP_KERNEL);
1026 if (!monc->monmap)
1027 return -ENOMEM;
1028 for (i = 0; i < num_mon; i++) {
1029 monc->monmap->mon_inst[i].addr = mon_addr[i];
1030 monc->monmap->mon_inst[i].addr.nonce = 0;
1031 monc->monmap->mon_inst[i].name.type =
1032 CEPH_ENTITY_TYPE_MON;
1033 monc->monmap->mon_inst[i].name.num = cpu_to_le64(i);
1034 }
1035 monc->monmap->num_mon = num_mon;
1036 return 0;
1037}
1038
1039int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl)
1040{
1041 int err = 0;
1042
1043 dout("init\n");
1044 memset(monc, 0, sizeof(*monc));
1045 monc->client = cl;
1046 monc->monmap = NULL;
1047 mutex_init(&monc->mutex);
1048
1049 err = build_initial_monmap(monc);
1050 if (err)
1051 goto out;
1052
1053 /* connection */
1054 /* authentication */
1055 monc->auth = ceph_auth_init(cl->options->name,
1056 cl->options->key);
1057 if (IS_ERR(monc->auth)) {
1058 err = PTR_ERR(monc->auth);
1059 goto out_monmap;
1060 }
1061 monc->auth->want_keys =
1062 CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |
1063 CEPH_ENTITY_TYPE_OSD | CEPH_ENTITY_TYPE_MDS;
1064
1065 /* msgs */
1066 err = -ENOMEM;
1067 monc->m_subscribe_ack = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE_ACK,
1068 sizeof(struct ceph_mon_subscribe_ack),
1069 GFP_KERNEL, true);
1070 if (!monc->m_subscribe_ack)
1071 goto out_auth;
1072
1073 monc->m_subscribe = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE, 128,
1074 GFP_KERNEL, true);
1075 if (!monc->m_subscribe)
1076 goto out_subscribe_ack;
1077
1078 monc->m_auth_reply = ceph_msg_new(CEPH_MSG_AUTH_REPLY, 4096,
1079 GFP_KERNEL, true);
1080 if (!monc->m_auth_reply)
1081 goto out_subscribe;
1082
1083 monc->m_auth = ceph_msg_new(CEPH_MSG_AUTH, 4096, GFP_KERNEL, true);
1084 monc->pending_auth = 0;
1085 if (!monc->m_auth)
1086 goto out_auth_reply;
1087
1088 ceph_con_init(&monc->con, monc, &mon_con_ops,
1089 &monc->client->msgr);
1090
1091 monc->cur_mon = -1;
1092 monc->had_a_connection = false;
1093 monc->hunt_mult = 1;
1094
1095 INIT_DELAYED_WORK(&monc->delayed_work, delayed_work);
1096 monc->generic_request_tree = RB_ROOT;
1097 monc->last_tid = 0;
1098
1099 monc->fs_cluster_id = CEPH_FS_CLUSTER_ID_NONE;
1100
1101 return 0;
1102
1103out_auth_reply:
1104 ceph_msg_put(monc->m_auth_reply);
1105out_subscribe:
1106 ceph_msg_put(monc->m_subscribe);
1107out_subscribe_ack:
1108 ceph_msg_put(monc->m_subscribe_ack);
1109out_auth:
1110 ceph_auth_destroy(monc->auth);
1111out_monmap:
1112 kfree(monc->monmap);
1113out:
1114 return err;
1115}
1116EXPORT_SYMBOL(ceph_monc_init);
1117
1118void ceph_monc_stop(struct ceph_mon_client *monc)
1119{
1120 dout("stop\n");
1121 cancel_delayed_work_sync(&monc->delayed_work);
1122
1123 mutex_lock(&monc->mutex);
1124 __close_session(monc);
1125 monc->cur_mon = -1;
1126 mutex_unlock(&monc->mutex);
1127
1128 /*
1129 * flush msgr queue before we destroy ourselves to ensure that:
1130 * - any work that references our embedded con is finished.
1131 * - any osd_client or other work that may reference an authorizer
1132 * finishes before we shut down the auth subsystem.
1133 */
1134 ceph_msgr_flush();
1135
1136 ceph_auth_destroy(monc->auth);
1137
1138 WARN_ON(!RB_EMPTY_ROOT(&monc->generic_request_tree));
1139
1140 ceph_msg_put(monc->m_auth);
1141 ceph_msg_put(monc->m_auth_reply);
1142 ceph_msg_put(monc->m_subscribe);
1143 ceph_msg_put(monc->m_subscribe_ack);
1144
1145 kfree(monc->monmap);
1146}
1147EXPORT_SYMBOL(ceph_monc_stop);
1148
1149static void finish_hunting(struct ceph_mon_client *monc)
1150{
1151 if (monc->hunting) {
1152 dout("%s found mon%d\n", __func__, monc->cur_mon);
1153 monc->hunting = false;
1154 monc->had_a_connection = true;
1155 un_backoff(monc);
1156 __schedule_delayed(monc);
1157 }
1158}
1159
1160static void handle_auth_reply(struct ceph_mon_client *monc,
1161 struct ceph_msg *msg)
1162{
1163 int ret;
1164 int was_auth = 0;
1165
1166 mutex_lock(&monc->mutex);
1167 was_auth = ceph_auth_is_authenticated(monc->auth);
1168 monc->pending_auth = 0;
1169 ret = ceph_handle_auth_reply(monc->auth, msg->front.iov_base,
1170 msg->front.iov_len,
1171 monc->m_auth->front.iov_base,
1172 monc->m_auth->front_alloc_len);
1173 if (ret > 0) {
1174 __send_prepared_auth_request(monc, ret);
1175 goto out;
1176 }
1177
1178 finish_hunting(monc);
1179
1180 if (ret < 0) {
1181 monc->client->auth_err = ret;
1182 } else if (!was_auth && ceph_auth_is_authenticated(monc->auth)) {
1183 dout("authenticated, starting session\n");
1184
1185 monc->client->msgr.inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
1186 monc->client->msgr.inst.name.num =
1187 cpu_to_le64(monc->auth->global_id);
1188
1189 __send_subscribe(monc);
1190 __resend_generic_request(monc);
1191
1192 pr_info("mon%d %s session established\n", monc->cur_mon,
1193 ceph_pr_addr(&monc->con.peer_addr));
1194 }
1195
1196out:
1197 mutex_unlock(&monc->mutex);
1198 if (monc->client->auth_err < 0)
1199 wake_up_all(&monc->client->auth_wq);
1200}
1201
1202static int __validate_auth(struct ceph_mon_client *monc)
1203{
1204 int ret;
1205
1206 if (monc->pending_auth)
1207 return 0;
1208
1209 ret = ceph_build_auth(monc->auth, monc->m_auth->front.iov_base,
1210 monc->m_auth->front_alloc_len);
1211 if (ret <= 0)
1212 return ret; /* either an error, or no need to authenticate */
1213 __send_prepared_auth_request(monc, ret);
1214 return 0;
1215}
1216
1217int ceph_monc_validate_auth(struct ceph_mon_client *monc)
1218{
1219 int ret;
1220
1221 mutex_lock(&monc->mutex);
1222 ret = __validate_auth(monc);
1223 mutex_unlock(&monc->mutex);
1224 return ret;
1225}
1226EXPORT_SYMBOL(ceph_monc_validate_auth);
1227
1228/*
1229 * handle incoming message
1230 */
1231static void dispatch(struct ceph_connection *con, struct ceph_msg *msg)
1232{
1233 struct ceph_mon_client *monc = con->private;
1234 int type = le16_to_cpu(msg->hdr.type);
1235
1236 if (!monc)
1237 return;
1238
1239 switch (type) {
1240 case CEPH_MSG_AUTH_REPLY:
1241 handle_auth_reply(monc, msg);
1242 break;
1243
1244 case CEPH_MSG_MON_SUBSCRIBE_ACK:
1245 handle_subscribe_ack(monc, msg);
1246 break;
1247
1248 case CEPH_MSG_STATFS_REPLY:
1249 handle_statfs_reply(monc, msg);
1250 break;
1251
1252 case CEPH_MSG_MON_GET_VERSION_REPLY:
1253 handle_get_version_reply(monc, msg);
1254 break;
1255
1256 case CEPH_MSG_MON_COMMAND_ACK:
1257 handle_command_ack(monc, msg);
1258 break;
1259
1260 case CEPH_MSG_MON_MAP:
1261 ceph_monc_handle_map(monc, msg);
1262 break;
1263
1264 case CEPH_MSG_OSD_MAP:
1265 ceph_osdc_handle_map(&monc->client->osdc, msg);
1266 break;
1267
1268 default:
1269 /* can the chained handler handle it? */
1270 if (monc->client->extra_mon_dispatch &&
1271 monc->client->extra_mon_dispatch(monc->client, msg) == 0)
1272 break;
1273
1274 pr_err("received unknown message type %d %s\n", type,
1275 ceph_msg_type_name(type));
1276 }
1277 ceph_msg_put(msg);
1278}
1279
1280/*
1281 * Allocate memory for incoming message
1282 */
1283static struct ceph_msg *mon_alloc_msg(struct ceph_connection *con,
1284 struct ceph_msg_header *hdr,
1285 int *skip)
1286{
1287 struct ceph_mon_client *monc = con->private;
1288 int type = le16_to_cpu(hdr->type);
1289 int front_len = le32_to_cpu(hdr->front_len);
1290 struct ceph_msg *m = NULL;
1291
1292 *skip = 0;
1293
1294 switch (type) {
1295 case CEPH_MSG_MON_SUBSCRIBE_ACK:
1296 m = ceph_msg_get(monc->m_subscribe_ack);
1297 break;
1298 case CEPH_MSG_STATFS_REPLY:
1299 case CEPH_MSG_MON_COMMAND_ACK:
1300 return get_generic_reply(con, hdr, skip);
1301 case CEPH_MSG_AUTH_REPLY:
1302 m = ceph_msg_get(monc->m_auth_reply);
1303 break;
1304 case CEPH_MSG_MON_GET_VERSION_REPLY:
1305 if (le64_to_cpu(hdr->tid) != 0)
1306 return get_generic_reply(con, hdr, skip);
1307
1308 /*
1309 * Older OSDs don't set reply tid even if the orignal
1310 * request had a non-zero tid. Work around this weirdness
1311 * by allocating a new message.
1312 */
1313 /* fall through */
1314 case CEPH_MSG_MON_MAP:
1315 case CEPH_MSG_MDS_MAP:
1316 case CEPH_MSG_OSD_MAP:
1317 case CEPH_MSG_FS_MAP_USER:
1318 m = ceph_msg_new(type, front_len, GFP_NOFS, false);
1319 if (!m)
1320 return NULL; /* ENOMEM--return skip == 0 */
1321 break;
1322 }
1323
1324 if (!m) {
1325 pr_info("alloc_msg unknown type %d\n", type);
1326 *skip = 1;
1327 } else if (front_len > m->front_alloc_len) {
1328 pr_warn("mon_alloc_msg front %d > prealloc %d (%u#%llu)\n",
1329 front_len, m->front_alloc_len,
1330 (unsigned int)con->peer_name.type,
1331 le64_to_cpu(con->peer_name.num));
1332 ceph_msg_put(m);
1333 m = ceph_msg_new(type, front_len, GFP_NOFS, false);
1334 }
1335
1336 return m;
1337}
1338
1339/*
1340 * If the monitor connection resets, pick a new monitor and resubmit
1341 * any pending requests.
1342 */
1343static void mon_fault(struct ceph_connection *con)
1344{
1345 struct ceph_mon_client *monc = con->private;
1346
1347 mutex_lock(&monc->mutex);
1348 dout("%s mon%d\n", __func__, monc->cur_mon);
1349 if (monc->cur_mon >= 0) {
1350 if (!monc->hunting) {
1351 dout("%s hunting for new mon\n", __func__);
1352 reopen_session(monc);
1353 __schedule_delayed(monc);
1354 } else {
1355 dout("%s already hunting\n", __func__);
1356 }
1357 }
1358 mutex_unlock(&monc->mutex);
1359}
1360
1361/*
1362 * We can ignore refcounting on the connection struct, as all references
1363 * will come from the messenger workqueue, which is drained prior to
1364 * mon_client destruction.
1365 */
1366static struct ceph_connection *con_get(struct ceph_connection *con)
1367{
1368 return con;
1369}
1370
1371static void con_put(struct ceph_connection *con)
1372{
1373}
1374
1375static const struct ceph_connection_operations mon_con_ops = {
1376 .get = con_get,
1377 .put = con_put,
1378 .dispatch = dispatch,
1379 .fault = mon_fault,
1380 .alloc_msg = mon_alloc_msg,
1381};