Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2// Copyright (C) 2022 Linutronix GmbH, John Ogness
3// Copyright (C) 2022 Intel, Thomas Gleixner
4
5#include <linux/kernel.h>
6#include <linux/console.h>
7#include <linux/delay.h>
8#include <linux/slab.h>
9#include "internal.h"
10/*
11 * Printk console printing implementation for consoles which does not depend
12 * on the legacy style console_lock mechanism.
13 *
14 * The state of the console is maintained in the "nbcon_state" atomic
15 * variable.
16 *
17 * The console is locked when:
18 *
19 * - The 'prio' field contains the priority of the context that owns the
20 * console. Only higher priority contexts are allowed to take over the
21 * lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked.
22 *
23 * - The 'cpu' field denotes on which CPU the console is locked. It is used
24 * to prevent busy waiting on the same CPU. Also it informs the lock owner
25 * that it has lost the lock in a more complex scenario when the lock was
26 * taken over by a higher priority context, released, and taken on another
27 * CPU with the same priority as the interrupted owner.
28 *
29 * The acquire mechanism uses a few more fields:
30 *
31 * - The 'req_prio' field is used by the handover approach to make the
32 * current owner aware that there is a context with a higher priority
33 * waiting for the friendly handover.
34 *
35 * - The 'unsafe' field allows to take over the console in a safe way in the
36 * middle of emitting a message. The field is set only when accessing some
37 * shared resources or when the console device is manipulated. It can be
38 * cleared, for example, after emitting one character when the console
39 * device is in a consistent state.
40 *
41 * - The 'unsafe_takeover' field is set when a hostile takeover took the
42 * console in an unsafe state. The console will stay in the unsafe state
43 * until re-initialized.
44 *
45 * The acquire mechanism uses three approaches:
46 *
47 * 1) Direct acquire when the console is not owned or is owned by a lower
48 * priority context and is in a safe state.
49 *
50 * 2) Friendly handover mechanism uses a request/grant handshake. It is used
51 * when the current owner has lower priority and the console is in an
52 * unsafe state.
53 *
54 * The requesting context:
55 *
56 * a) Sets its priority into the 'req_prio' field.
57 *
58 * b) Waits (with a timeout) for the owning context to unlock the
59 * console.
60 *
61 * c) Takes the lock and clears the 'req_prio' field.
62 *
63 * The owning context:
64 *
65 * a) Observes the 'req_prio' field set on exit from the unsafe
66 * console state.
67 *
68 * b) Gives up console ownership by clearing the 'prio' field.
69 *
70 * 3) Unsafe hostile takeover allows to take over the lock even when the
71 * console is an unsafe state. It is used only in panic() by the final
72 * attempt to flush consoles in a try and hope mode.
73 *
74 * Note that separate record buffers are used in panic(). As a result,
75 * the messages can be read and formatted without any risk even after
76 * using the hostile takeover in unsafe state.
77 *
78 * The release function simply clears the 'prio' field.
79 *
80 * All operations on @console::nbcon_state are atomic cmpxchg based to
81 * handle concurrency.
82 *
83 * The acquire/release functions implement only minimal policies:
84 *
85 * - Preference for higher priority contexts.
86 * - Protection of the panic CPU.
87 *
88 * All other policy decisions must be made at the call sites:
89 *
90 * - What is marked as an unsafe section.
91 * - Whether to spin-wait if there is already an owner and the console is
92 * in an unsafe state.
93 * - Whether to attempt an unsafe hostile takeover.
94 *
95 * The design allows to implement the well known:
96 *
97 * acquire()
98 * output_one_printk_record()
99 * release()
100 *
101 * The output of one printk record might be interrupted with a higher priority
102 * context. The new owner is supposed to reprint the entire interrupted record
103 * from scratch.
104 */
105
106/**
107 * nbcon_state_set - Helper function to set the console state
108 * @con: Console to update
109 * @new: The new state to write
110 *
111 * Only to be used when the console is not yet or no longer visible in the
112 * system. Otherwise use nbcon_state_try_cmpxchg().
113 */
114static inline void nbcon_state_set(struct console *con, struct nbcon_state *new)
115{
116 atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom);
117}
118
119/**
120 * nbcon_state_read - Helper function to read the console state
121 * @con: Console to read
122 * @state: The state to store the result
123 */
124static inline void nbcon_state_read(struct console *con, struct nbcon_state *state)
125{
126 state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state));
127}
128
129/**
130 * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state
131 * @con: Console to update
132 * @cur: Old/expected state
133 * @new: New state
134 *
135 * Return: True on success. False on fail and @cur is updated.
136 */
137static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur,
138 struct nbcon_state *new)
139{
140 return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom);
141}
142
143#ifdef CONFIG_64BIT
144
145#define __seq_to_nbcon_seq(seq) (seq)
146#define __nbcon_seq_to_seq(seq) (seq)
147
148#else /* CONFIG_64BIT */
149
150#define __seq_to_nbcon_seq(seq) ((u32)seq)
151
152static inline u64 __nbcon_seq_to_seq(u32 nbcon_seq)
153{
154 u64 seq;
155 u64 rb_next_seq;
156
157 /*
158 * The provided sequence is only the lower 32 bits of the ringbuffer
159 * sequence. It needs to be expanded to 64bit. Get the next sequence
160 * number from the ringbuffer and fold it.
161 *
162 * Having a 32bit representation in the console is sufficient.
163 * If a console ever gets more than 2^31 records behind
164 * the ringbuffer then this is the least of the problems.
165 *
166 * Also the access to the ring buffer is always safe.
167 */
168 rb_next_seq = prb_next_seq(prb);
169 seq = rb_next_seq - ((u32)rb_next_seq - nbcon_seq);
170
171 return seq;
172}
173
174#endif /* CONFIG_64BIT */
175
176/**
177 * nbcon_seq_read - Read the current console sequence
178 * @con: Console to read the sequence of
179 *
180 * Return: Sequence number of the next record to print on @con.
181 */
182u64 nbcon_seq_read(struct console *con)
183{
184 unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq));
185
186 return __nbcon_seq_to_seq(nbcon_seq);
187}
188
189/**
190 * nbcon_seq_force - Force console sequence to a specific value
191 * @con: Console to work on
192 * @seq: Sequence number value to set
193 *
194 * Only to be used during init (before registration) or in extreme situations
195 * (such as panic with CONSOLE_REPLAY_ALL).
196 */
197void nbcon_seq_force(struct console *con, u64 seq)
198{
199 /*
200 * If the specified record no longer exists, the oldest available record
201 * is chosen. This is especially important on 32bit systems because only
202 * the lower 32 bits of the sequence number are stored. The upper 32 bits
203 * are derived from the sequence numbers available in the ringbuffer.
204 */
205 u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb));
206
207 atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __seq_to_nbcon_seq(valid_seq));
208
209 /* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */
210 con->seq = 0;
211}
212
213/**
214 * nbcon_seq_try_update - Try to update the console sequence number
215 * @ctxt: Pointer to an acquire context that contains
216 * all information about the acquire mode
217 * @new_seq: The new sequence number to set
218 *
219 * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to
220 * the 64bit value). This could be a different value than @new_seq if
221 * nbcon_seq_force() was used or the current context no longer owns the
222 * console. In the later case, it will stop printing anyway.
223 */
224static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq)
225{
226 unsigned long nbcon_seq = __seq_to_nbcon_seq(ctxt->seq);
227 struct console *con = ctxt->console;
228
229 if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq,
230 __seq_to_nbcon_seq(new_seq))) {
231 ctxt->seq = new_seq;
232 } else {
233 ctxt->seq = nbcon_seq_read(con);
234 }
235}
236
237/**
238 * nbcon_context_try_acquire_direct - Try to acquire directly
239 * @ctxt: The context of the caller
240 * @cur: The current console state
241 *
242 * Acquire the console when it is released. Also acquire the console when
243 * the current owner has a lower priority and the console is in a safe state.
244 *
245 * Return: 0 on success. Otherwise, an error code on failure. Also @cur
246 * is updated to the latest state when failed to modify it.
247 *
248 * Errors:
249 *
250 * -EPERM: A panic is in progress and this is not the panic CPU.
251 * Or the current owner or waiter has the same or higher
252 * priority. No acquire method can be successful in
253 * this case.
254 *
255 * -EBUSY: The current owner has a lower priority but the console
256 * in an unsafe state. The caller should try using
257 * the handover acquire method.
258 */
259static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt,
260 struct nbcon_state *cur)
261{
262 unsigned int cpu = smp_processor_id();
263 struct console *con = ctxt->console;
264 struct nbcon_state new;
265
266 do {
267 if (other_cpu_in_panic())
268 return -EPERM;
269
270 if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio)
271 return -EPERM;
272
273 if (cur->unsafe)
274 return -EBUSY;
275
276 /*
277 * The console should never be safe for a direct acquire
278 * if an unsafe hostile takeover has ever happened.
279 */
280 WARN_ON_ONCE(cur->unsafe_takeover);
281
282 new.atom = cur->atom;
283 new.prio = ctxt->prio;
284 new.req_prio = NBCON_PRIO_NONE;
285 new.unsafe = cur->unsafe_takeover;
286 new.cpu = cpu;
287
288 } while (!nbcon_state_try_cmpxchg(con, cur, &new));
289
290 return 0;
291}
292
293static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio)
294{
295 /*
296 * The request context is well defined by the @req_prio because:
297 *
298 * - Only a context with a higher priority can take over the request.
299 * - There are only three priorities.
300 * - Only one CPU is allowed to request PANIC priority.
301 * - Lower priorities are ignored during panic() until reboot.
302 *
303 * As a result, the following scenario is *not* possible:
304 *
305 * 1. Another context with a higher priority directly takes ownership.
306 * 2. The higher priority context releases the ownership.
307 * 3. A lower priority context takes the ownership.
308 * 4. Another context with the same priority as this context
309 * creates a request and starts waiting.
310 */
311
312 return (cur->req_prio == expected_prio);
313}
314
315/**
316 * nbcon_context_try_acquire_requested - Try to acquire after having
317 * requested a handover
318 * @ctxt: The context of the caller
319 * @cur: The current console state
320 *
321 * This is a helper function for nbcon_context_try_acquire_handover().
322 * It is called when the console is in an unsafe state. The current
323 * owner will release the console on exit from the unsafe region.
324 *
325 * Return: 0 on success and @cur is updated to the new console state.
326 * Otherwise an error code on failure.
327 *
328 * Errors:
329 *
330 * -EPERM: A panic is in progress and this is not the panic CPU
331 * or this context is no longer the waiter.
332 *
333 * -EBUSY: The console is still locked. The caller should
334 * continue waiting.
335 *
336 * Note: The caller must still remove the request when an error has occurred
337 * except when this context is no longer the waiter.
338 */
339static int nbcon_context_try_acquire_requested(struct nbcon_context *ctxt,
340 struct nbcon_state *cur)
341{
342 unsigned int cpu = smp_processor_id();
343 struct console *con = ctxt->console;
344 struct nbcon_state new;
345
346 /* Note that the caller must still remove the request! */
347 if (other_cpu_in_panic())
348 return -EPERM;
349
350 /*
351 * Note that the waiter will also change if there was an unsafe
352 * hostile takeover.
353 */
354 if (!nbcon_waiter_matches(cur, ctxt->prio))
355 return -EPERM;
356
357 /* If still locked, caller should continue waiting. */
358 if (cur->prio != NBCON_PRIO_NONE)
359 return -EBUSY;
360
361 /*
362 * The previous owner should have never released ownership
363 * in an unsafe region.
364 */
365 WARN_ON_ONCE(cur->unsafe);
366
367 new.atom = cur->atom;
368 new.prio = ctxt->prio;
369 new.req_prio = NBCON_PRIO_NONE;
370 new.unsafe = cur->unsafe_takeover;
371 new.cpu = cpu;
372
373 if (!nbcon_state_try_cmpxchg(con, cur, &new)) {
374 /*
375 * The acquire could fail only when it has been taken
376 * over by a higher priority context.
377 */
378 WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio));
379 return -EPERM;
380 }
381
382 /* Handover success. This context now owns the console. */
383 return 0;
384}
385
386/**
387 * nbcon_context_try_acquire_handover - Try to acquire via handover
388 * @ctxt: The context of the caller
389 * @cur: The current console state
390 *
391 * The function must be called only when the context has higher priority
392 * than the current owner and the console is in an unsafe state.
393 * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY.
394 *
395 * The function sets "req_prio" field to make the current owner aware of
396 * the request. Then it waits until the current owner releases the console,
397 * or an even higher context takes over the request, or timeout expires.
398 *
399 * The current owner checks the "req_prio" field on exit from the unsafe
400 * region and releases the console. It does not touch the "req_prio" field
401 * so that the console stays reserved for the waiter.
402 *
403 * Return: 0 on success. Otherwise, an error code on failure. Also @cur
404 * is updated to the latest state when failed to modify it.
405 *
406 * Errors:
407 *
408 * -EPERM: A panic is in progress and this is not the panic CPU.
409 * Or a higher priority context has taken over the
410 * console or the handover request.
411 *
412 * -EBUSY: The current owner is on the same CPU so that the hand
413 * shake could not work. Or the current owner is not
414 * willing to wait (zero timeout). Or the console does
415 * not enter the safe state before timeout passed. The
416 * caller might still use the unsafe hostile takeover
417 * when allowed.
418 *
419 * -EAGAIN: @cur has changed when creating the handover request.
420 * The caller should retry with direct acquire.
421 */
422static int nbcon_context_try_acquire_handover(struct nbcon_context *ctxt,
423 struct nbcon_state *cur)
424{
425 unsigned int cpu = smp_processor_id();
426 struct console *con = ctxt->console;
427 struct nbcon_state new;
428 int timeout;
429 int request_err = -EBUSY;
430
431 /*
432 * Check that the handover is called when the direct acquire failed
433 * with -EBUSY.
434 */
435 WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
436 WARN_ON_ONCE(!cur->unsafe);
437
438 /* Handover is not possible on the same CPU. */
439 if (cur->cpu == cpu)
440 return -EBUSY;
441
442 /*
443 * Console stays unsafe after an unsafe takeover until re-initialized.
444 * Waiting is not going to help in this case.
445 */
446 if (cur->unsafe_takeover)
447 return -EBUSY;
448
449 /* Is the caller willing to wait? */
450 if (ctxt->spinwait_max_us == 0)
451 return -EBUSY;
452
453 /*
454 * Setup a request for the handover. The caller should try to acquire
455 * the console directly when the current state has been modified.
456 */
457 new.atom = cur->atom;
458 new.req_prio = ctxt->prio;
459 if (!nbcon_state_try_cmpxchg(con, cur, &new))
460 return -EAGAIN;
461
462 cur->atom = new.atom;
463
464 /* Wait until there is no owner and then acquire the console. */
465 for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) {
466 /* On successful acquire, this request is cleared. */
467 request_err = nbcon_context_try_acquire_requested(ctxt, cur);
468 if (!request_err)
469 return 0;
470
471 /*
472 * If the acquire should be aborted, it must be ensured
473 * that the request is removed before returning to caller.
474 */
475 if (request_err == -EPERM)
476 break;
477
478 udelay(1);
479
480 /* Re-read the state because some time has passed. */
481 nbcon_state_read(con, cur);
482 }
483
484 /* Timed out or aborted. Carefully remove handover request. */
485 do {
486 /*
487 * No need to remove request if there is a new waiter. This
488 * can only happen if a higher priority context has taken over
489 * the console or the handover request.
490 */
491 if (!nbcon_waiter_matches(cur, ctxt->prio))
492 return -EPERM;
493
494 /* Unset request for handover. */
495 new.atom = cur->atom;
496 new.req_prio = NBCON_PRIO_NONE;
497 if (nbcon_state_try_cmpxchg(con, cur, &new)) {
498 /*
499 * Request successfully unset. Report failure of
500 * acquiring via handover.
501 */
502 cur->atom = new.atom;
503 return request_err;
504 }
505
506 /*
507 * Unable to remove request. Try to acquire in case
508 * the owner has released the lock.
509 */
510 } while (nbcon_context_try_acquire_requested(ctxt, cur));
511
512 /* Lucky timing. The acquire succeeded while removing the request. */
513 return 0;
514}
515
516/**
517 * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover
518 * @ctxt: The context of the caller
519 * @cur: The current console state
520 *
521 * Acquire the console even in the unsafe state.
522 *
523 * It can be permitted by setting the 'allow_unsafe_takeover' field only
524 * by the final attempt to flush messages in panic().
525 *
526 * Return: 0 on success. -EPERM when not allowed by the context.
527 */
528static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt,
529 struct nbcon_state *cur)
530{
531 unsigned int cpu = smp_processor_id();
532 struct console *con = ctxt->console;
533 struct nbcon_state new;
534
535 if (!ctxt->allow_unsafe_takeover)
536 return -EPERM;
537
538 /* Ensure caller is allowed to perform unsafe hostile takeovers. */
539 if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC))
540 return -EPERM;
541
542 /*
543 * Check that try_acquire_direct() and try_acquire_handover() returned
544 * -EBUSY in the right situation.
545 */
546 WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
547 WARN_ON_ONCE(cur->unsafe != true);
548
549 do {
550 new.atom = cur->atom;
551 new.cpu = cpu;
552 new.prio = ctxt->prio;
553 new.unsafe |= cur->unsafe_takeover;
554 new.unsafe_takeover |= cur->unsafe;
555
556 } while (!nbcon_state_try_cmpxchg(con, cur, &new));
557
558 return 0;
559}
560
561static struct printk_buffers panic_nbcon_pbufs;
562
563/**
564 * nbcon_context_try_acquire - Try to acquire nbcon console
565 * @ctxt: The context of the caller
566 *
567 * Return: True if the console was acquired. False otherwise.
568 *
569 * If the caller allowed an unsafe hostile takeover, on success the
570 * caller should check the current console state to see if it is
571 * in an unsafe state. Otherwise, on success the caller may assume
572 * the console is not in an unsafe state.
573 */
574__maybe_unused
575static bool nbcon_context_try_acquire(struct nbcon_context *ctxt)
576{
577 unsigned int cpu = smp_processor_id();
578 struct console *con = ctxt->console;
579 struct nbcon_state cur;
580 int err;
581
582 nbcon_state_read(con, &cur);
583try_again:
584 err = nbcon_context_try_acquire_direct(ctxt, &cur);
585 if (err != -EBUSY)
586 goto out;
587
588 err = nbcon_context_try_acquire_handover(ctxt, &cur);
589 if (err == -EAGAIN)
590 goto try_again;
591 if (err != -EBUSY)
592 goto out;
593
594 err = nbcon_context_try_acquire_hostile(ctxt, &cur);
595out:
596 if (err)
597 return false;
598
599 /* Acquire succeeded. */
600
601 /* Assign the appropriate buffer for this context. */
602 if (atomic_read(&panic_cpu) == cpu)
603 ctxt->pbufs = &panic_nbcon_pbufs;
604 else
605 ctxt->pbufs = con->pbufs;
606
607 /* Set the record sequence for this context to print. */
608 ctxt->seq = nbcon_seq_read(ctxt->console);
609
610 return true;
611}
612
613static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu,
614 int expected_prio)
615{
616 /*
617 * Since consoles can only be acquired by higher priorities,
618 * owning contexts are uniquely identified by @prio. However,
619 * since contexts can unexpectedly lose ownership, it is
620 * possible that later another owner appears with the same
621 * priority. For this reason @cpu is also needed.
622 */
623
624 if (cur->prio != expected_prio)
625 return false;
626
627 if (cur->cpu != expected_cpu)
628 return false;
629
630 return true;
631}
632
633/**
634 * nbcon_context_release - Release the console
635 * @ctxt: The nbcon context from nbcon_context_try_acquire()
636 */
637static void nbcon_context_release(struct nbcon_context *ctxt)
638{
639 unsigned int cpu = smp_processor_id();
640 struct console *con = ctxt->console;
641 struct nbcon_state cur;
642 struct nbcon_state new;
643
644 nbcon_state_read(con, &cur);
645
646 do {
647 if (!nbcon_owner_matches(&cur, cpu, ctxt->prio))
648 break;
649
650 new.atom = cur.atom;
651 new.prio = NBCON_PRIO_NONE;
652
653 /*
654 * If @unsafe_takeover is set, it is kept set so that
655 * the state remains permanently unsafe.
656 */
657 new.unsafe |= cur.unsafe_takeover;
658
659 } while (!nbcon_state_try_cmpxchg(con, &cur, &new));
660
661 ctxt->pbufs = NULL;
662}
663
664/**
665 * nbcon_context_can_proceed - Check whether ownership can proceed
666 * @ctxt: The nbcon context from nbcon_context_try_acquire()
667 * @cur: The current console state
668 *
669 * Return: True if this context still owns the console. False if
670 * ownership was handed over or taken.
671 *
672 * Must be invoked when entering the unsafe state to make sure that it still
673 * owns the lock. Also must be invoked when exiting the unsafe context
674 * to eventually free the lock for a higher priority context which asked
675 * for the friendly handover.
676 *
677 * It can be called inside an unsafe section when the console is just
678 * temporary in safe state instead of exiting and entering the unsafe
679 * state.
680 *
681 * Also it can be called in the safe context before doing an expensive
682 * safe operation. It does not make sense to do the operation when
683 * a higher priority context took the lock.
684 *
685 * When this function returns false then the calling context no longer owns
686 * the console and is no longer allowed to go forward. In this case it must
687 * back out immediately and carefully. The buffer content is also no longer
688 * trusted since it no longer belongs to the calling context.
689 */
690static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_state *cur)
691{
692 unsigned int cpu = smp_processor_id();
693
694 /* Make sure this context still owns the console. */
695 if (!nbcon_owner_matches(cur, cpu, ctxt->prio))
696 return false;
697
698 /* The console owner can proceed if there is no waiter. */
699 if (cur->req_prio == NBCON_PRIO_NONE)
700 return true;
701
702 /*
703 * A console owner within an unsafe region is always allowed to
704 * proceed, even if there are waiters. It can perform a handover
705 * when exiting the unsafe region. Otherwise the waiter will
706 * need to perform an unsafe hostile takeover.
707 */
708 if (cur->unsafe)
709 return true;
710
711 /* Waiters always have higher priorities than owners. */
712 WARN_ON_ONCE(cur->req_prio <= cur->prio);
713
714 /*
715 * Having a safe point for take over and eventually a few
716 * duplicated characters or a full line is way better than a
717 * hostile takeover. Post processing can take care of the garbage.
718 * Release and hand over.
719 */
720 nbcon_context_release(ctxt);
721
722 /*
723 * It is not clear whether the waiter really took over ownership. The
724 * outermost callsite must make the final decision whether console
725 * ownership is needed for it to proceed. If yes, it must reacquire
726 * ownership (possibly hostile) before carefully proceeding.
727 *
728 * The calling context no longer owns the console so go back all the
729 * way instead of trying to implement reacquire heuristics in tons of
730 * places.
731 */
732 return false;
733}
734
735/**
736 * nbcon_can_proceed - Check whether ownership can proceed
737 * @wctxt: The write context that was handed to the write function
738 *
739 * Return: True if this context still owns the console. False if
740 * ownership was handed over or taken.
741 *
742 * It is used in nbcon_enter_unsafe() to make sure that it still owns the
743 * lock. Also it is used in nbcon_exit_unsafe() to eventually free the lock
744 * for a higher priority context which asked for the friendly handover.
745 *
746 * It can be called inside an unsafe section when the console is just
747 * temporary in safe state instead of exiting and entering the unsafe state.
748 *
749 * Also it can be called in the safe context before doing an expensive safe
750 * operation. It does not make sense to do the operation when a higher
751 * priority context took the lock.
752 *
753 * When this function returns false then the calling context no longer owns
754 * the console and is no longer allowed to go forward. In this case it must
755 * back out immediately and carefully. The buffer content is also no longer
756 * trusted since it no longer belongs to the calling context.
757 */
758bool nbcon_can_proceed(struct nbcon_write_context *wctxt)
759{
760 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
761 struct console *con = ctxt->console;
762 struct nbcon_state cur;
763
764 nbcon_state_read(con, &cur);
765
766 return nbcon_context_can_proceed(ctxt, &cur);
767}
768EXPORT_SYMBOL_GPL(nbcon_can_proceed);
769
770#define nbcon_context_enter_unsafe(c) __nbcon_context_update_unsafe(c, true)
771#define nbcon_context_exit_unsafe(c) __nbcon_context_update_unsafe(c, false)
772
773/**
774 * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state
775 * @ctxt: The nbcon context from nbcon_context_try_acquire()
776 * @unsafe: The new value for the unsafe bit
777 *
778 * Return: True if the unsafe state was updated and this context still
779 * owns the console. Otherwise false if ownership was handed
780 * over or taken.
781 *
782 * This function allows console owners to modify the unsafe status of the
783 * console.
784 *
785 * When this function returns false then the calling context no longer owns
786 * the console and is no longer allowed to go forward. In this case it must
787 * back out immediately and carefully. The buffer content is also no longer
788 * trusted since it no longer belongs to the calling context.
789 *
790 * Internal helper to avoid duplicated code.
791 */
792static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe)
793{
794 struct console *con = ctxt->console;
795 struct nbcon_state cur;
796 struct nbcon_state new;
797
798 nbcon_state_read(con, &cur);
799
800 do {
801 /*
802 * The unsafe bit must not be cleared if an
803 * unsafe hostile takeover has occurred.
804 */
805 if (!unsafe && cur.unsafe_takeover)
806 goto out;
807
808 if (!nbcon_context_can_proceed(ctxt, &cur))
809 return false;
810
811 new.atom = cur.atom;
812 new.unsafe = unsafe;
813 } while (!nbcon_state_try_cmpxchg(con, &cur, &new));
814
815 cur.atom = new.atom;
816out:
817 return nbcon_context_can_proceed(ctxt, &cur);
818}
819
820/**
821 * nbcon_enter_unsafe - Enter an unsafe region in the driver
822 * @wctxt: The write context that was handed to the write function
823 *
824 * Return: True if this context still owns the console. False if
825 * ownership was handed over or taken.
826 *
827 * When this function returns false then the calling context no longer owns
828 * the console and is no longer allowed to go forward. In this case it must
829 * back out immediately and carefully. The buffer content is also no longer
830 * trusted since it no longer belongs to the calling context.
831 */
832bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt)
833{
834 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
835
836 return nbcon_context_enter_unsafe(ctxt);
837}
838EXPORT_SYMBOL_GPL(nbcon_enter_unsafe);
839
840/**
841 * nbcon_exit_unsafe - Exit an unsafe region in the driver
842 * @wctxt: The write context that was handed to the write function
843 *
844 * Return: True if this context still owns the console. False if
845 * ownership was handed over or taken.
846 *
847 * When this function returns false then the calling context no longer owns
848 * the console and is no longer allowed to go forward. In this case it must
849 * back out immediately and carefully. The buffer content is also no longer
850 * trusted since it no longer belongs to the calling context.
851 */
852bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt)
853{
854 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
855
856 return nbcon_context_exit_unsafe(ctxt);
857}
858EXPORT_SYMBOL_GPL(nbcon_exit_unsafe);
859
860/**
861 * nbcon_emit_next_record - Emit a record in the acquired context
862 * @wctxt: The write context that will be handed to the write function
863 *
864 * Return: True if this context still owns the console. False if
865 * ownership was handed over or taken.
866 *
867 * When this function returns false then the calling context no longer owns
868 * the console and is no longer allowed to go forward. In this case it must
869 * back out immediately and carefully. The buffer content is also no longer
870 * trusted since it no longer belongs to the calling context. If the caller
871 * wants to do more it must reacquire the console first.
872 *
873 * When true is returned, @wctxt->ctxt.backlog indicates whether there are
874 * still records pending in the ringbuffer,
875 */
876__maybe_unused
877static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt)
878{
879 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
880 struct console *con = ctxt->console;
881 bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
882 struct printk_message pmsg = {
883 .pbufs = ctxt->pbufs,
884 };
885 unsigned long con_dropped;
886 struct nbcon_state cur;
887 unsigned long dropped;
888 bool done;
889
890 /*
891 * The printk buffers are filled within an unsafe section. This
892 * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from
893 * clobbering each other.
894 */
895
896 if (!nbcon_context_enter_unsafe(ctxt))
897 return false;
898
899 ctxt->backlog = printk_get_next_message(&pmsg, ctxt->seq, is_extended, true);
900 if (!ctxt->backlog)
901 return nbcon_context_exit_unsafe(ctxt);
902
903 /*
904 * @con->dropped is not protected in case of an unsafe hostile
905 * takeover. In that situation the update can be racy so
906 * annotate it accordingly.
907 */
908 con_dropped = data_race(READ_ONCE(con->dropped));
909
910 dropped = con_dropped + pmsg.dropped;
911 if (dropped && !is_extended)
912 console_prepend_dropped(&pmsg, dropped);
913
914 if (!nbcon_context_exit_unsafe(ctxt))
915 return false;
916
917 /* For skipped records just update seq/dropped in @con. */
918 if (pmsg.outbuf_len == 0)
919 goto update_con;
920
921 /* Initialize the write context for driver callbacks. */
922 wctxt->outbuf = &pmsg.pbufs->outbuf[0];
923 wctxt->len = pmsg.outbuf_len;
924 nbcon_state_read(con, &cur);
925 wctxt->unsafe_takeover = cur.unsafe_takeover;
926
927 if (con->write_atomic) {
928 done = con->write_atomic(con, wctxt);
929 } else {
930 nbcon_context_release(ctxt);
931 WARN_ON_ONCE(1);
932 done = false;
933 }
934
935 /* If not done, the emit was aborted. */
936 if (!done)
937 return false;
938
939 /*
940 * Since any dropped message was successfully output, reset the
941 * dropped count for the console.
942 */
943 dropped = 0;
944update_con:
945 /*
946 * The dropped count and the sequence number are updated within an
947 * unsafe section. This limits update races to the panic context and
948 * allows the panic context to win.
949 */
950
951 if (!nbcon_context_enter_unsafe(ctxt))
952 return false;
953
954 if (dropped != con_dropped) {
955 /* Counterpart to the READ_ONCE() above. */
956 WRITE_ONCE(con->dropped, dropped);
957 }
958
959 nbcon_seq_try_update(ctxt, pmsg.seq + 1);
960
961 return nbcon_context_exit_unsafe(ctxt);
962}
963
964/**
965 * nbcon_alloc - Allocate buffers needed by the nbcon console
966 * @con: Console to allocate buffers for
967 *
968 * Return: True on success. False otherwise and the console cannot
969 * be used.
970 *
971 * This is not part of nbcon_init() because buffer allocation must
972 * be performed earlier in the console registration process.
973 */
974bool nbcon_alloc(struct console *con)
975{
976 if (con->flags & CON_BOOT) {
977 /*
978 * Boot console printing is synchronized with legacy console
979 * printing, so boot consoles can share the same global printk
980 * buffers.
981 */
982 con->pbufs = &printk_shared_pbufs;
983 } else {
984 con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL);
985 if (!con->pbufs) {
986 con_printk(KERN_ERR, con, "failed to allocate printing buffer\n");
987 return false;
988 }
989 }
990
991 return true;
992}
993
994/**
995 * nbcon_init - Initialize the nbcon console specific data
996 * @con: Console to initialize
997 *
998 * nbcon_alloc() *must* be called and succeed before this function
999 * is called.
1000 *
1001 * This function expects that the legacy @con->seq has been set.
1002 */
1003void nbcon_init(struct console *con)
1004{
1005 struct nbcon_state state = { };
1006
1007 /* nbcon_alloc() must have been called and successful! */
1008 BUG_ON(!con->pbufs);
1009
1010 nbcon_seq_force(con, con->seq);
1011 nbcon_state_set(con, &state);
1012}
1013
1014/**
1015 * nbcon_free - Free and cleanup the nbcon console specific data
1016 * @con: Console to free/cleanup nbcon data
1017 */
1018void nbcon_free(struct console *con)
1019{
1020 struct nbcon_state state = { };
1021
1022 nbcon_state_set(con, &state);
1023
1024 /* Boot consoles share global printk buffers. */
1025 if (!(con->flags & CON_BOOT))
1026 kfree(con->pbufs);
1027
1028 con->pbufs = NULL;
1029}
1// SPDX-License-Identifier: GPL-2.0-only
2// Copyright (C) 2022 Linutronix GmbH, John Ogness
3// Copyright (C) 2022 Intel, Thomas Gleixner
4
5#include <linux/kernel.h>
6#include <linux/console.h>
7#include <linux/delay.h>
8#include <linux/slab.h>
9#include "internal.h"
10/*
11 * Printk console printing implementation for consoles which does not depend
12 * on the legacy style console_lock mechanism.
13 *
14 * The state of the console is maintained in the "nbcon_state" atomic
15 * variable.
16 *
17 * The console is locked when:
18 *
19 * - The 'prio' field contains the priority of the context that owns the
20 * console. Only higher priority contexts are allowed to take over the
21 * lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked.
22 *
23 * - The 'cpu' field denotes on which CPU the console is locked. It is used
24 * to prevent busy waiting on the same CPU. Also it informs the lock owner
25 * that it has lost the lock in a more complex scenario when the lock was
26 * taken over by a higher priority context, released, and taken on another
27 * CPU with the same priority as the interrupted owner.
28 *
29 * The acquire mechanism uses a few more fields:
30 *
31 * - The 'req_prio' field is used by the handover approach to make the
32 * current owner aware that there is a context with a higher priority
33 * waiting for the friendly handover.
34 *
35 * - The 'unsafe' field allows to take over the console in a safe way in the
36 * middle of emitting a message. The field is set only when accessing some
37 * shared resources or when the console device is manipulated. It can be
38 * cleared, for example, after emitting one character when the console
39 * device is in a consistent state.
40 *
41 * - The 'unsafe_takeover' field is set when a hostile takeover took the
42 * console in an unsafe state. The console will stay in the unsafe state
43 * until re-initialized.
44 *
45 * The acquire mechanism uses three approaches:
46 *
47 * 1) Direct acquire when the console is not owned or is owned by a lower
48 * priority context and is in a safe state.
49 *
50 * 2) Friendly handover mechanism uses a request/grant handshake. It is used
51 * when the current owner has lower priority and the console is in an
52 * unsafe state.
53 *
54 * The requesting context:
55 *
56 * a) Sets its priority into the 'req_prio' field.
57 *
58 * b) Waits (with a timeout) for the owning context to unlock the
59 * console.
60 *
61 * c) Takes the lock and clears the 'req_prio' field.
62 *
63 * The owning context:
64 *
65 * a) Observes the 'req_prio' field set on exit from the unsafe
66 * console state.
67 *
68 * b) Gives up console ownership by clearing the 'prio' field.
69 *
70 * 3) Unsafe hostile takeover allows to take over the lock even when the
71 * console is an unsafe state. It is used only in panic() by the final
72 * attempt to flush consoles in a try and hope mode.
73 *
74 * Note that separate record buffers are used in panic(). As a result,
75 * the messages can be read and formatted without any risk even after
76 * using the hostile takeover in unsafe state.
77 *
78 * The release function simply clears the 'prio' field.
79 *
80 * All operations on @console::nbcon_state are atomic cmpxchg based to
81 * handle concurrency.
82 *
83 * The acquire/release functions implement only minimal policies:
84 *
85 * - Preference for higher priority contexts.
86 * - Protection of the panic CPU.
87 *
88 * All other policy decisions must be made at the call sites:
89 *
90 * - What is marked as an unsafe section.
91 * - Whether to spin-wait if there is already an owner and the console is
92 * in an unsafe state.
93 * - Whether to attempt an unsafe hostile takeover.
94 *
95 * The design allows to implement the well known:
96 *
97 * acquire()
98 * output_one_printk_record()
99 * release()
100 *
101 * The output of one printk record might be interrupted with a higher priority
102 * context. The new owner is supposed to reprint the entire interrupted record
103 * from scratch.
104 */
105
106/**
107 * nbcon_state_set - Helper function to set the console state
108 * @con: Console to update
109 * @new: The new state to write
110 *
111 * Only to be used when the console is not yet or no longer visible in the
112 * system. Otherwise use nbcon_state_try_cmpxchg().
113 */
114static inline void nbcon_state_set(struct console *con, struct nbcon_state *new)
115{
116 atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom);
117}
118
119/**
120 * nbcon_state_read - Helper function to read the console state
121 * @con: Console to read
122 * @state: The state to store the result
123 */
124static inline void nbcon_state_read(struct console *con, struct nbcon_state *state)
125{
126 state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state));
127}
128
129/**
130 * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state
131 * @con: Console to update
132 * @cur: Old/expected state
133 * @new: New state
134 *
135 * Return: True on success. False on fail and @cur is updated.
136 */
137static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur,
138 struct nbcon_state *new)
139{
140 return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom);
141}
142
143/**
144 * nbcon_seq_read - Read the current console sequence
145 * @con: Console to read the sequence of
146 *
147 * Return: Sequence number of the next record to print on @con.
148 */
149u64 nbcon_seq_read(struct console *con)
150{
151 unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq));
152
153 return __ulseq_to_u64seq(prb, nbcon_seq);
154}
155
156/**
157 * nbcon_seq_force - Force console sequence to a specific value
158 * @con: Console to work on
159 * @seq: Sequence number value to set
160 *
161 * Only to be used during init (before registration) or in extreme situations
162 * (such as panic with CONSOLE_REPLAY_ALL).
163 */
164void nbcon_seq_force(struct console *con, u64 seq)
165{
166 /*
167 * If the specified record no longer exists, the oldest available record
168 * is chosen. This is especially important on 32bit systems because only
169 * the lower 32 bits of the sequence number are stored. The upper 32 bits
170 * are derived from the sequence numbers available in the ringbuffer.
171 */
172 u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb));
173
174 atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq));
175
176 /* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */
177 con->seq = 0;
178}
179
180/**
181 * nbcon_seq_try_update - Try to update the console sequence number
182 * @ctxt: Pointer to an acquire context that contains
183 * all information about the acquire mode
184 * @new_seq: The new sequence number to set
185 *
186 * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to
187 * the 64bit value). This could be a different value than @new_seq if
188 * nbcon_seq_force() was used or the current context no longer owns the
189 * console. In the later case, it will stop printing anyway.
190 */
191static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq)
192{
193 unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq);
194 struct console *con = ctxt->console;
195
196 if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq,
197 __u64seq_to_ulseq(new_seq))) {
198 ctxt->seq = new_seq;
199 } else {
200 ctxt->seq = nbcon_seq_read(con);
201 }
202}
203
204/**
205 * nbcon_context_try_acquire_direct - Try to acquire directly
206 * @ctxt: The context of the caller
207 * @cur: The current console state
208 *
209 * Acquire the console when it is released. Also acquire the console when
210 * the current owner has a lower priority and the console is in a safe state.
211 *
212 * Return: 0 on success. Otherwise, an error code on failure. Also @cur
213 * is updated to the latest state when failed to modify it.
214 *
215 * Errors:
216 *
217 * -EPERM: A panic is in progress and this is not the panic CPU.
218 * Or the current owner or waiter has the same or higher
219 * priority. No acquire method can be successful in
220 * this case.
221 *
222 * -EBUSY: The current owner has a lower priority but the console
223 * in an unsafe state. The caller should try using
224 * the handover acquire method.
225 */
226static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt,
227 struct nbcon_state *cur)
228{
229 unsigned int cpu = smp_processor_id();
230 struct console *con = ctxt->console;
231 struct nbcon_state new;
232
233 do {
234 if (other_cpu_in_panic())
235 return -EPERM;
236
237 if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio)
238 return -EPERM;
239
240 if (cur->unsafe)
241 return -EBUSY;
242
243 /*
244 * The console should never be safe for a direct acquire
245 * if an unsafe hostile takeover has ever happened.
246 */
247 WARN_ON_ONCE(cur->unsafe_takeover);
248
249 new.atom = cur->atom;
250 new.prio = ctxt->prio;
251 new.req_prio = NBCON_PRIO_NONE;
252 new.unsafe = cur->unsafe_takeover;
253 new.cpu = cpu;
254
255 } while (!nbcon_state_try_cmpxchg(con, cur, &new));
256
257 return 0;
258}
259
260static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio)
261{
262 /*
263 * The request context is well defined by the @req_prio because:
264 *
265 * - Only a context with a higher priority can take over the request.
266 * - There are only three priorities.
267 * - Only one CPU is allowed to request PANIC priority.
268 * - Lower priorities are ignored during panic() until reboot.
269 *
270 * As a result, the following scenario is *not* possible:
271 *
272 * 1. Another context with a higher priority directly takes ownership.
273 * 2. The higher priority context releases the ownership.
274 * 3. A lower priority context takes the ownership.
275 * 4. Another context with the same priority as this context
276 * creates a request and starts waiting.
277 */
278
279 return (cur->req_prio == expected_prio);
280}
281
282/**
283 * nbcon_context_try_acquire_requested - Try to acquire after having
284 * requested a handover
285 * @ctxt: The context of the caller
286 * @cur: The current console state
287 *
288 * This is a helper function for nbcon_context_try_acquire_handover().
289 * It is called when the console is in an unsafe state. The current
290 * owner will release the console on exit from the unsafe region.
291 *
292 * Return: 0 on success and @cur is updated to the new console state.
293 * Otherwise an error code on failure.
294 *
295 * Errors:
296 *
297 * -EPERM: A panic is in progress and this is not the panic CPU
298 * or this context is no longer the waiter.
299 *
300 * -EBUSY: The console is still locked. The caller should
301 * continue waiting.
302 *
303 * Note: The caller must still remove the request when an error has occurred
304 * except when this context is no longer the waiter.
305 */
306static int nbcon_context_try_acquire_requested(struct nbcon_context *ctxt,
307 struct nbcon_state *cur)
308{
309 unsigned int cpu = smp_processor_id();
310 struct console *con = ctxt->console;
311 struct nbcon_state new;
312
313 /* Note that the caller must still remove the request! */
314 if (other_cpu_in_panic())
315 return -EPERM;
316
317 /*
318 * Note that the waiter will also change if there was an unsafe
319 * hostile takeover.
320 */
321 if (!nbcon_waiter_matches(cur, ctxt->prio))
322 return -EPERM;
323
324 /* If still locked, caller should continue waiting. */
325 if (cur->prio != NBCON_PRIO_NONE)
326 return -EBUSY;
327
328 /*
329 * The previous owner should have never released ownership
330 * in an unsafe region.
331 */
332 WARN_ON_ONCE(cur->unsafe);
333
334 new.atom = cur->atom;
335 new.prio = ctxt->prio;
336 new.req_prio = NBCON_PRIO_NONE;
337 new.unsafe = cur->unsafe_takeover;
338 new.cpu = cpu;
339
340 if (!nbcon_state_try_cmpxchg(con, cur, &new)) {
341 /*
342 * The acquire could fail only when it has been taken
343 * over by a higher priority context.
344 */
345 WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio));
346 return -EPERM;
347 }
348
349 /* Handover success. This context now owns the console. */
350 return 0;
351}
352
353/**
354 * nbcon_context_try_acquire_handover - Try to acquire via handover
355 * @ctxt: The context of the caller
356 * @cur: The current console state
357 *
358 * The function must be called only when the context has higher priority
359 * than the current owner and the console is in an unsafe state.
360 * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY.
361 *
362 * The function sets "req_prio" field to make the current owner aware of
363 * the request. Then it waits until the current owner releases the console,
364 * or an even higher context takes over the request, or timeout expires.
365 *
366 * The current owner checks the "req_prio" field on exit from the unsafe
367 * region and releases the console. It does not touch the "req_prio" field
368 * so that the console stays reserved for the waiter.
369 *
370 * Return: 0 on success. Otherwise, an error code on failure. Also @cur
371 * is updated to the latest state when failed to modify it.
372 *
373 * Errors:
374 *
375 * -EPERM: A panic is in progress and this is not the panic CPU.
376 * Or a higher priority context has taken over the
377 * console or the handover request.
378 *
379 * -EBUSY: The current owner is on the same CPU so that the hand
380 * shake could not work. Or the current owner is not
381 * willing to wait (zero timeout). Or the console does
382 * not enter the safe state before timeout passed. The
383 * caller might still use the unsafe hostile takeover
384 * when allowed.
385 *
386 * -EAGAIN: @cur has changed when creating the handover request.
387 * The caller should retry with direct acquire.
388 */
389static int nbcon_context_try_acquire_handover(struct nbcon_context *ctxt,
390 struct nbcon_state *cur)
391{
392 unsigned int cpu = smp_processor_id();
393 struct console *con = ctxt->console;
394 struct nbcon_state new;
395 int timeout;
396 int request_err = -EBUSY;
397
398 /*
399 * Check that the handover is called when the direct acquire failed
400 * with -EBUSY.
401 */
402 WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
403 WARN_ON_ONCE(!cur->unsafe);
404
405 /* Handover is not possible on the same CPU. */
406 if (cur->cpu == cpu)
407 return -EBUSY;
408
409 /*
410 * Console stays unsafe after an unsafe takeover until re-initialized.
411 * Waiting is not going to help in this case.
412 */
413 if (cur->unsafe_takeover)
414 return -EBUSY;
415
416 /* Is the caller willing to wait? */
417 if (ctxt->spinwait_max_us == 0)
418 return -EBUSY;
419
420 /*
421 * Setup a request for the handover. The caller should try to acquire
422 * the console directly when the current state has been modified.
423 */
424 new.atom = cur->atom;
425 new.req_prio = ctxt->prio;
426 if (!nbcon_state_try_cmpxchg(con, cur, &new))
427 return -EAGAIN;
428
429 cur->atom = new.atom;
430
431 /* Wait until there is no owner and then acquire the console. */
432 for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) {
433 /* On successful acquire, this request is cleared. */
434 request_err = nbcon_context_try_acquire_requested(ctxt, cur);
435 if (!request_err)
436 return 0;
437
438 /*
439 * If the acquire should be aborted, it must be ensured
440 * that the request is removed before returning to caller.
441 */
442 if (request_err == -EPERM)
443 break;
444
445 udelay(1);
446
447 /* Re-read the state because some time has passed. */
448 nbcon_state_read(con, cur);
449 }
450
451 /* Timed out or aborted. Carefully remove handover request. */
452 do {
453 /*
454 * No need to remove request if there is a new waiter. This
455 * can only happen if a higher priority context has taken over
456 * the console or the handover request.
457 */
458 if (!nbcon_waiter_matches(cur, ctxt->prio))
459 return -EPERM;
460
461 /* Unset request for handover. */
462 new.atom = cur->atom;
463 new.req_prio = NBCON_PRIO_NONE;
464 if (nbcon_state_try_cmpxchg(con, cur, &new)) {
465 /*
466 * Request successfully unset. Report failure of
467 * acquiring via handover.
468 */
469 cur->atom = new.atom;
470 return request_err;
471 }
472
473 /*
474 * Unable to remove request. Try to acquire in case
475 * the owner has released the lock.
476 */
477 } while (nbcon_context_try_acquire_requested(ctxt, cur));
478
479 /* Lucky timing. The acquire succeeded while removing the request. */
480 return 0;
481}
482
483/**
484 * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover
485 * @ctxt: The context of the caller
486 * @cur: The current console state
487 *
488 * Acquire the console even in the unsafe state.
489 *
490 * It can be permitted by setting the 'allow_unsafe_takeover' field only
491 * by the final attempt to flush messages in panic().
492 *
493 * Return: 0 on success. -EPERM when not allowed by the context.
494 */
495static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt,
496 struct nbcon_state *cur)
497{
498 unsigned int cpu = smp_processor_id();
499 struct console *con = ctxt->console;
500 struct nbcon_state new;
501
502 if (!ctxt->allow_unsafe_takeover)
503 return -EPERM;
504
505 /* Ensure caller is allowed to perform unsafe hostile takeovers. */
506 if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC))
507 return -EPERM;
508
509 /*
510 * Check that try_acquire_direct() and try_acquire_handover() returned
511 * -EBUSY in the right situation.
512 */
513 WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
514 WARN_ON_ONCE(cur->unsafe != true);
515
516 do {
517 new.atom = cur->atom;
518 new.cpu = cpu;
519 new.prio = ctxt->prio;
520 new.unsafe |= cur->unsafe_takeover;
521 new.unsafe_takeover |= cur->unsafe;
522
523 } while (!nbcon_state_try_cmpxchg(con, cur, &new));
524
525 return 0;
526}
527
528static struct printk_buffers panic_nbcon_pbufs;
529
530/**
531 * nbcon_context_try_acquire - Try to acquire nbcon console
532 * @ctxt: The context of the caller
533 *
534 * Return: True if the console was acquired. False otherwise.
535 *
536 * If the caller allowed an unsafe hostile takeover, on success the
537 * caller should check the current console state to see if it is
538 * in an unsafe state. Otherwise, on success the caller may assume
539 * the console is not in an unsafe state.
540 */
541__maybe_unused
542static bool nbcon_context_try_acquire(struct nbcon_context *ctxt)
543{
544 unsigned int cpu = smp_processor_id();
545 struct console *con = ctxt->console;
546 struct nbcon_state cur;
547 int err;
548
549 nbcon_state_read(con, &cur);
550try_again:
551 err = nbcon_context_try_acquire_direct(ctxt, &cur);
552 if (err != -EBUSY)
553 goto out;
554
555 err = nbcon_context_try_acquire_handover(ctxt, &cur);
556 if (err == -EAGAIN)
557 goto try_again;
558 if (err != -EBUSY)
559 goto out;
560
561 err = nbcon_context_try_acquire_hostile(ctxt, &cur);
562out:
563 if (err)
564 return false;
565
566 /* Acquire succeeded. */
567
568 /* Assign the appropriate buffer for this context. */
569 if (atomic_read(&panic_cpu) == cpu)
570 ctxt->pbufs = &panic_nbcon_pbufs;
571 else
572 ctxt->pbufs = con->pbufs;
573
574 /* Set the record sequence for this context to print. */
575 ctxt->seq = nbcon_seq_read(ctxt->console);
576
577 return true;
578}
579
580static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu,
581 int expected_prio)
582{
583 /*
584 * Since consoles can only be acquired by higher priorities,
585 * owning contexts are uniquely identified by @prio. However,
586 * since contexts can unexpectedly lose ownership, it is
587 * possible that later another owner appears with the same
588 * priority. For this reason @cpu is also needed.
589 */
590
591 if (cur->prio != expected_prio)
592 return false;
593
594 if (cur->cpu != expected_cpu)
595 return false;
596
597 return true;
598}
599
600/**
601 * nbcon_context_release - Release the console
602 * @ctxt: The nbcon context from nbcon_context_try_acquire()
603 */
604static void nbcon_context_release(struct nbcon_context *ctxt)
605{
606 unsigned int cpu = smp_processor_id();
607 struct console *con = ctxt->console;
608 struct nbcon_state cur;
609 struct nbcon_state new;
610
611 nbcon_state_read(con, &cur);
612
613 do {
614 if (!nbcon_owner_matches(&cur, cpu, ctxt->prio))
615 break;
616
617 new.atom = cur.atom;
618 new.prio = NBCON_PRIO_NONE;
619
620 /*
621 * If @unsafe_takeover is set, it is kept set so that
622 * the state remains permanently unsafe.
623 */
624 new.unsafe |= cur.unsafe_takeover;
625
626 } while (!nbcon_state_try_cmpxchg(con, &cur, &new));
627
628 ctxt->pbufs = NULL;
629}
630
631/**
632 * nbcon_context_can_proceed - Check whether ownership can proceed
633 * @ctxt: The nbcon context from nbcon_context_try_acquire()
634 * @cur: The current console state
635 *
636 * Return: True if this context still owns the console. False if
637 * ownership was handed over or taken.
638 *
639 * Must be invoked when entering the unsafe state to make sure that it still
640 * owns the lock. Also must be invoked when exiting the unsafe context
641 * to eventually free the lock for a higher priority context which asked
642 * for the friendly handover.
643 *
644 * It can be called inside an unsafe section when the console is just
645 * temporary in safe state instead of exiting and entering the unsafe
646 * state.
647 *
648 * Also it can be called in the safe context before doing an expensive
649 * safe operation. It does not make sense to do the operation when
650 * a higher priority context took the lock.
651 *
652 * When this function returns false then the calling context no longer owns
653 * the console and is no longer allowed to go forward. In this case it must
654 * back out immediately and carefully. The buffer content is also no longer
655 * trusted since it no longer belongs to the calling context.
656 */
657static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_state *cur)
658{
659 unsigned int cpu = smp_processor_id();
660
661 /* Make sure this context still owns the console. */
662 if (!nbcon_owner_matches(cur, cpu, ctxt->prio))
663 return false;
664
665 /* The console owner can proceed if there is no waiter. */
666 if (cur->req_prio == NBCON_PRIO_NONE)
667 return true;
668
669 /*
670 * A console owner within an unsafe region is always allowed to
671 * proceed, even if there are waiters. It can perform a handover
672 * when exiting the unsafe region. Otherwise the waiter will
673 * need to perform an unsafe hostile takeover.
674 */
675 if (cur->unsafe)
676 return true;
677
678 /* Waiters always have higher priorities than owners. */
679 WARN_ON_ONCE(cur->req_prio <= cur->prio);
680
681 /*
682 * Having a safe point for take over and eventually a few
683 * duplicated characters or a full line is way better than a
684 * hostile takeover. Post processing can take care of the garbage.
685 * Release and hand over.
686 */
687 nbcon_context_release(ctxt);
688
689 /*
690 * It is not clear whether the waiter really took over ownership. The
691 * outermost callsite must make the final decision whether console
692 * ownership is needed for it to proceed. If yes, it must reacquire
693 * ownership (possibly hostile) before carefully proceeding.
694 *
695 * The calling context no longer owns the console so go back all the
696 * way instead of trying to implement reacquire heuristics in tons of
697 * places.
698 */
699 return false;
700}
701
702/**
703 * nbcon_can_proceed - Check whether ownership can proceed
704 * @wctxt: The write context that was handed to the write function
705 *
706 * Return: True if this context still owns the console. False if
707 * ownership was handed over or taken.
708 *
709 * It is used in nbcon_enter_unsafe() to make sure that it still owns the
710 * lock. Also it is used in nbcon_exit_unsafe() to eventually free the lock
711 * for a higher priority context which asked for the friendly handover.
712 *
713 * It can be called inside an unsafe section when the console is just
714 * temporary in safe state instead of exiting and entering the unsafe state.
715 *
716 * Also it can be called in the safe context before doing an expensive safe
717 * operation. It does not make sense to do the operation when a higher
718 * priority context took the lock.
719 *
720 * When this function returns false then the calling context no longer owns
721 * the console and is no longer allowed to go forward. In this case it must
722 * back out immediately and carefully. The buffer content is also no longer
723 * trusted since it no longer belongs to the calling context.
724 */
725bool nbcon_can_proceed(struct nbcon_write_context *wctxt)
726{
727 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
728 struct console *con = ctxt->console;
729 struct nbcon_state cur;
730
731 nbcon_state_read(con, &cur);
732
733 return nbcon_context_can_proceed(ctxt, &cur);
734}
735EXPORT_SYMBOL_GPL(nbcon_can_proceed);
736
737#define nbcon_context_enter_unsafe(c) __nbcon_context_update_unsafe(c, true)
738#define nbcon_context_exit_unsafe(c) __nbcon_context_update_unsafe(c, false)
739
740/**
741 * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state
742 * @ctxt: The nbcon context from nbcon_context_try_acquire()
743 * @unsafe: The new value for the unsafe bit
744 *
745 * Return: True if the unsafe state was updated and this context still
746 * owns the console. Otherwise false if ownership was handed
747 * over or taken.
748 *
749 * This function allows console owners to modify the unsafe status of the
750 * console.
751 *
752 * When this function returns false then the calling context no longer owns
753 * the console and is no longer allowed to go forward. In this case it must
754 * back out immediately and carefully. The buffer content is also no longer
755 * trusted since it no longer belongs to the calling context.
756 *
757 * Internal helper to avoid duplicated code.
758 */
759static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe)
760{
761 struct console *con = ctxt->console;
762 struct nbcon_state cur;
763 struct nbcon_state new;
764
765 nbcon_state_read(con, &cur);
766
767 do {
768 /*
769 * The unsafe bit must not be cleared if an
770 * unsafe hostile takeover has occurred.
771 */
772 if (!unsafe && cur.unsafe_takeover)
773 goto out;
774
775 if (!nbcon_context_can_proceed(ctxt, &cur))
776 return false;
777
778 new.atom = cur.atom;
779 new.unsafe = unsafe;
780 } while (!nbcon_state_try_cmpxchg(con, &cur, &new));
781
782 cur.atom = new.atom;
783out:
784 return nbcon_context_can_proceed(ctxt, &cur);
785}
786
787/**
788 * nbcon_enter_unsafe - Enter an unsafe region in the driver
789 * @wctxt: The write context that was handed to the write function
790 *
791 * Return: True if this context still owns the console. False if
792 * ownership was handed over or taken.
793 *
794 * When this function returns false then the calling context no longer owns
795 * the console and is no longer allowed to go forward. In this case it must
796 * back out immediately and carefully. The buffer content is also no longer
797 * trusted since it no longer belongs to the calling context.
798 */
799bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt)
800{
801 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
802
803 return nbcon_context_enter_unsafe(ctxt);
804}
805EXPORT_SYMBOL_GPL(nbcon_enter_unsafe);
806
807/**
808 * nbcon_exit_unsafe - Exit an unsafe region in the driver
809 * @wctxt: The write context that was handed to the write function
810 *
811 * Return: True if this context still owns the console. False if
812 * ownership was handed over or taken.
813 *
814 * When this function returns false then the calling context no longer owns
815 * the console and is no longer allowed to go forward. In this case it must
816 * back out immediately and carefully. The buffer content is also no longer
817 * trusted since it no longer belongs to the calling context.
818 */
819bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt)
820{
821 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
822
823 return nbcon_context_exit_unsafe(ctxt);
824}
825EXPORT_SYMBOL_GPL(nbcon_exit_unsafe);
826
827/**
828 * nbcon_emit_next_record - Emit a record in the acquired context
829 * @wctxt: The write context that will be handed to the write function
830 *
831 * Return: True if this context still owns the console. False if
832 * ownership was handed over or taken.
833 *
834 * When this function returns false then the calling context no longer owns
835 * the console and is no longer allowed to go forward. In this case it must
836 * back out immediately and carefully. The buffer content is also no longer
837 * trusted since it no longer belongs to the calling context. If the caller
838 * wants to do more it must reacquire the console first.
839 *
840 * When true is returned, @wctxt->ctxt.backlog indicates whether there are
841 * still records pending in the ringbuffer,
842 */
843__maybe_unused
844static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt)
845{
846 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
847 struct console *con = ctxt->console;
848 bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
849 struct printk_message pmsg = {
850 .pbufs = ctxt->pbufs,
851 };
852 unsigned long con_dropped;
853 struct nbcon_state cur;
854 unsigned long dropped;
855 bool done;
856
857 /*
858 * The printk buffers are filled within an unsafe section. This
859 * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from
860 * clobbering each other.
861 */
862
863 if (!nbcon_context_enter_unsafe(ctxt))
864 return false;
865
866 ctxt->backlog = printk_get_next_message(&pmsg, ctxt->seq, is_extended, true);
867 if (!ctxt->backlog)
868 return nbcon_context_exit_unsafe(ctxt);
869
870 /*
871 * @con->dropped is not protected in case of an unsafe hostile
872 * takeover. In that situation the update can be racy so
873 * annotate it accordingly.
874 */
875 con_dropped = data_race(READ_ONCE(con->dropped));
876
877 dropped = con_dropped + pmsg.dropped;
878 if (dropped && !is_extended)
879 console_prepend_dropped(&pmsg, dropped);
880
881 if (!nbcon_context_exit_unsafe(ctxt))
882 return false;
883
884 /* For skipped records just update seq/dropped in @con. */
885 if (pmsg.outbuf_len == 0)
886 goto update_con;
887
888 /* Initialize the write context for driver callbacks. */
889 wctxt->outbuf = &pmsg.pbufs->outbuf[0];
890 wctxt->len = pmsg.outbuf_len;
891 nbcon_state_read(con, &cur);
892 wctxt->unsafe_takeover = cur.unsafe_takeover;
893
894 if (con->write_atomic) {
895 done = con->write_atomic(con, wctxt);
896 } else {
897 nbcon_context_release(ctxt);
898 WARN_ON_ONCE(1);
899 done = false;
900 }
901
902 /* If not done, the emit was aborted. */
903 if (!done)
904 return false;
905
906 /*
907 * Since any dropped message was successfully output, reset the
908 * dropped count for the console.
909 */
910 dropped = 0;
911update_con:
912 /*
913 * The dropped count and the sequence number are updated within an
914 * unsafe section. This limits update races to the panic context and
915 * allows the panic context to win.
916 */
917
918 if (!nbcon_context_enter_unsafe(ctxt))
919 return false;
920
921 if (dropped != con_dropped) {
922 /* Counterpart to the READ_ONCE() above. */
923 WRITE_ONCE(con->dropped, dropped);
924 }
925
926 nbcon_seq_try_update(ctxt, pmsg.seq + 1);
927
928 return nbcon_context_exit_unsafe(ctxt);
929}
930
931/**
932 * nbcon_alloc - Allocate buffers needed by the nbcon console
933 * @con: Console to allocate buffers for
934 *
935 * Return: True on success. False otherwise and the console cannot
936 * be used.
937 *
938 * This is not part of nbcon_init() because buffer allocation must
939 * be performed earlier in the console registration process.
940 */
941bool nbcon_alloc(struct console *con)
942{
943 if (con->flags & CON_BOOT) {
944 /*
945 * Boot console printing is synchronized with legacy console
946 * printing, so boot consoles can share the same global printk
947 * buffers.
948 */
949 con->pbufs = &printk_shared_pbufs;
950 } else {
951 con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL);
952 if (!con->pbufs) {
953 con_printk(KERN_ERR, con, "failed to allocate printing buffer\n");
954 return false;
955 }
956 }
957
958 return true;
959}
960
961/**
962 * nbcon_init - Initialize the nbcon console specific data
963 * @con: Console to initialize
964 *
965 * nbcon_alloc() *must* be called and succeed before this function
966 * is called.
967 *
968 * This function expects that the legacy @con->seq has been set.
969 */
970void nbcon_init(struct console *con)
971{
972 struct nbcon_state state = { };
973
974 /* nbcon_alloc() must have been called and successful! */
975 BUG_ON(!con->pbufs);
976
977 nbcon_seq_force(con, con->seq);
978 nbcon_state_set(con, &state);
979}
980
981/**
982 * nbcon_free - Free and cleanup the nbcon console specific data
983 * @con: Console to free/cleanup nbcon data
984 */
985void nbcon_free(struct console *con)
986{
987 struct nbcon_state state = { };
988
989 nbcon_state_set(con, &state);
990
991 /* Boot consoles share global printk buffers. */
992 if (!(con->flags & CON_BOOT))
993 kfree(con->pbufs);
994
995 con->pbufs = NULL;
996}