1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *
  4 * MMC software queue support based on command queue interfaces
  5 *
  6 * Copyright (C) 2019 Linaro, Inc.
  7 * Author: Baolin Wang <baolin.wang@linaro.org>
  8 */
  9
 10#include <linux/mmc/card.h>
 11#include <linux/mmc/host.h>
 12#include <linux/module.h>
 13
 14#include "mmc_hsq.h"
 15
 16static void mmc_hsq_retry_handler(struct work_struct *work)
 17{
 18	struct mmc_hsq *hsq = container_of(work, struct mmc_hsq, retry_work);
 19	struct mmc_host *mmc = hsq->mmc;
 20
 21	mmc->ops->request(mmc, hsq->mrq);
 22}
 23
 24static void mmc_hsq_pump_requests(struct mmc_hsq *hsq)
 25{
 26	struct mmc_host *mmc = hsq->mmc;
 27	struct hsq_slot *slot;
 28	unsigned long flags;
 29	int ret = 0;
 30
 31	spin_lock_irqsave(&hsq->lock, flags);
 32
 33	/* Make sure we are not already running a request now */
 34	if (hsq->mrq || hsq->recovery_halt) {
 35		spin_unlock_irqrestore(&hsq->lock, flags);
 36		return;
 37	}
 38
 39	/* Make sure there are remain requests need to pump */
 40	if (!hsq->qcnt || !hsq->enabled) {
 41		spin_unlock_irqrestore(&hsq->lock, flags);
 42		return;
 43	}
 44
 45	slot = &hsq->slot[hsq->next_tag];
 46	hsq->mrq = slot->mrq;
 47	hsq->qcnt--;
 48
 49	spin_unlock_irqrestore(&hsq->lock, flags);
 50
 51	if (mmc->ops->request_atomic)
 52		ret = mmc->ops->request_atomic(mmc, hsq->mrq);
 53	else
 54		mmc->ops->request(mmc, hsq->mrq);
 55
 56	/*
 57	 * If returning BUSY from request_atomic(), which means the card
 58	 * may be busy now, and we should change to non-atomic context to
 59	 * try again for this unusual case, to avoid time-consuming operations
 60	 * in the atomic context.
 61	 *
 62	 * Note: we just give a warning for other error cases, since the host
 63	 * driver will handle them.
 64	 */
 65	if (ret == -EBUSY)
 66		schedule_work(&hsq->retry_work);
 67	else
 68		WARN_ON_ONCE(ret);
 69}
 70
 71static void mmc_hsq_update_next_tag(struct mmc_hsq *hsq, int remains)
 72{
 73	int tag;
 74
 75	/*
 76	 * If there are no remain requests in software queue, then set a invalid
 77	 * tag.
 78	 */
 79	if (!remains) {
 80		hsq->next_tag = HSQ_INVALID_TAG;
 81		hsq->tail_tag = HSQ_INVALID_TAG;
 82		return;
 83	}
 84
 85	tag = hsq->tag_slot[hsq->next_tag];
 86	hsq->tag_slot[hsq->next_tag] = HSQ_INVALID_TAG;
 87	hsq->next_tag = tag;
 88}
 89
 90static void mmc_hsq_post_request(struct mmc_hsq *hsq)
 91{
 92	unsigned long flags;
 93	int remains;
 94
 95	spin_lock_irqsave(&hsq->lock, flags);
 96
 97	remains = hsq->qcnt;
 98	hsq->mrq = NULL;
 99
100	/* Update the next available tag to be queued. */
101	mmc_hsq_update_next_tag(hsq, remains);
102
103	if (hsq->waiting_for_idle && !remains) {
104		hsq->waiting_for_idle = false;
105		wake_up(&hsq->wait_queue);
106	}
107
108	/* Do not pump new request in recovery mode. */
109	if (hsq->recovery_halt) {
110		spin_unlock_irqrestore(&hsq->lock, flags);
111		return;
112	}
113
114	spin_unlock_irqrestore(&hsq->lock, flags);
115
116	 /*
117	  * Try to pump new request to host controller as fast as possible,
118	  * after completing previous request.
119	  */
120	if (remains > 0)
121		mmc_hsq_pump_requests(hsq);
122}
123
124/**
125 * mmc_hsq_finalize_request - finalize one request if the request is done
126 * @mmc: the host controller
127 * @mrq: the request need to be finalized
128 *
129 * Return true if we finalized the corresponding request in software queue,
130 * otherwise return false.
131 */
132bool mmc_hsq_finalize_request(struct mmc_host *mmc, struct mmc_request *mrq)
133{
134	struct mmc_hsq *hsq = mmc->cqe_private;
135	unsigned long flags;
136
137	spin_lock_irqsave(&hsq->lock, flags);
138
139	if (!hsq->enabled || !hsq->mrq || hsq->mrq != mrq) {
140		spin_unlock_irqrestore(&hsq->lock, flags);
141		return false;
142	}
143
144	/*
145	 * Clear current completed slot request to make a room for new request.
146	 */
147	hsq->slot[hsq->next_tag].mrq = NULL;
148
149	spin_unlock_irqrestore(&hsq->lock, flags);
150
151	mmc_cqe_request_done(mmc, hsq->mrq);
152
153	mmc_hsq_post_request(hsq);
154
155	return true;
156}
157EXPORT_SYMBOL_GPL(mmc_hsq_finalize_request);
158
159static void mmc_hsq_recovery_start(struct mmc_host *mmc)
160{
161	struct mmc_hsq *hsq = mmc->cqe_private;
162	unsigned long flags;
163
164	spin_lock_irqsave(&hsq->lock, flags);
165
166	hsq->recovery_halt = true;
167
168	spin_unlock_irqrestore(&hsq->lock, flags);
169}
170
171static void mmc_hsq_recovery_finish(struct mmc_host *mmc)
172{
173	struct mmc_hsq *hsq = mmc->cqe_private;
174	int remains;
175
176	spin_lock_irq(&hsq->lock);
177
178	hsq->recovery_halt = false;
179	remains = hsq->qcnt;
180
181	spin_unlock_irq(&hsq->lock);
182
183	/*
184	 * Try to pump new request if there are request pending in software
185	 * queue after finishing recovery.
186	 */
187	if (remains > 0)
188		mmc_hsq_pump_requests(hsq);
189}
190
191static int mmc_hsq_request(struct mmc_host *mmc, struct mmc_request *mrq)
192{
193	struct mmc_hsq *hsq = mmc->cqe_private;
194	int tag = mrq->tag;
195
196	spin_lock_irq(&hsq->lock);
197
198	if (!hsq->enabled) {
199		spin_unlock_irq(&hsq->lock);
200		return -ESHUTDOWN;
201	}
202
203	/* Do not queue any new requests in recovery mode. */
204	if (hsq->recovery_halt) {
205		spin_unlock_irq(&hsq->lock);
206		return -EBUSY;
207	}
208
209	hsq->slot[tag].mrq = mrq;
210
211	/*
212	 * Set the next tag as current request tag if no available
213	 * next tag.
214	 */
215	if (hsq->next_tag == HSQ_INVALID_TAG) {
216		hsq->next_tag = tag;
217		hsq->tail_tag = tag;
218		hsq->tag_slot[hsq->tail_tag] = HSQ_INVALID_TAG;
219	} else {
220		hsq->tag_slot[hsq->tail_tag] = tag;
221		hsq->tail_tag = tag;
222	}
223
224	hsq->qcnt++;
225
226	spin_unlock_irq(&hsq->lock);
227
228	mmc_hsq_pump_requests(hsq);
229
230	return 0;
231}
232
233static void mmc_hsq_post_req(struct mmc_host *mmc, struct mmc_request *mrq)
234{
235	if (mmc->ops->post_req)
236		mmc->ops->post_req(mmc, mrq, 0);
237}
238
239static bool mmc_hsq_queue_is_idle(struct mmc_hsq *hsq, int *ret)
240{
241	bool is_idle;
242
243	spin_lock_irq(&hsq->lock);
244
245	is_idle = (!hsq->mrq && !hsq->qcnt) ||
246		hsq->recovery_halt;
247
248	*ret = hsq->recovery_halt ? -EBUSY : 0;
249	hsq->waiting_for_idle = !is_idle;
250
251	spin_unlock_irq(&hsq->lock);
252
253	return is_idle;
254}
255
256static int mmc_hsq_wait_for_idle(struct mmc_host *mmc)
257{
258	struct mmc_hsq *hsq = mmc->cqe_private;
259	int ret;
260
261	wait_event(hsq->wait_queue,
262		   mmc_hsq_queue_is_idle(hsq, &ret));
263
264	return ret;
265}
266
267static void mmc_hsq_disable(struct mmc_host *mmc)
268{
269	struct mmc_hsq *hsq = mmc->cqe_private;
270	u32 timeout = 500;
271	int ret;
272
273	spin_lock_irq(&hsq->lock);
274
275	if (!hsq->enabled) {
276		spin_unlock_irq(&hsq->lock);
277		return;
278	}
279
280	spin_unlock_irq(&hsq->lock);
281
282	ret = wait_event_timeout(hsq->wait_queue,
283				 mmc_hsq_queue_is_idle(hsq, &ret),
284				 msecs_to_jiffies(timeout));
285	if (ret == 0) {
286		pr_warn("could not stop mmc software queue\n");
287		return;
288	}
289
290	spin_lock_irq(&hsq->lock);
291
292	hsq->enabled = false;
293
294	spin_unlock_irq(&hsq->lock);
295}
296
297static int mmc_hsq_enable(struct mmc_host *mmc, struct mmc_card *card)
298{
299	struct mmc_hsq *hsq = mmc->cqe_private;
300
301	spin_lock_irq(&hsq->lock);
302
303	if (hsq->enabled) {
304		spin_unlock_irq(&hsq->lock);
305		return -EBUSY;
306	}
307
308	hsq->enabled = true;
309
310	spin_unlock_irq(&hsq->lock);
311
312	return 0;
313}
314
315static const struct mmc_cqe_ops mmc_hsq_ops = {
316	.cqe_enable = mmc_hsq_enable,
317	.cqe_disable = mmc_hsq_disable,
318	.cqe_request = mmc_hsq_request,
319	.cqe_post_req = mmc_hsq_post_req,
320	.cqe_wait_for_idle = mmc_hsq_wait_for_idle,
321	.cqe_recovery_start = mmc_hsq_recovery_start,
322	.cqe_recovery_finish = mmc_hsq_recovery_finish,
323};
324
325int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc)
326{
327	int i;
328	hsq->num_slots = HSQ_NUM_SLOTS;
329	hsq->next_tag = HSQ_INVALID_TAG;
330	hsq->tail_tag = HSQ_INVALID_TAG;
331
332	hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots,
333				 sizeof(struct hsq_slot), GFP_KERNEL);
334	if (!hsq->slot)
335		return -ENOMEM;
336
337	hsq->mmc = mmc;
338	hsq->mmc->cqe_private = hsq;
339	mmc->cqe_ops = &mmc_hsq_ops;
340
341	for (i = 0; i < HSQ_NUM_SLOTS; i++)
342		hsq->tag_slot[i] = HSQ_INVALID_TAG;
343
344	INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler);
345	spin_lock_init(&hsq->lock);
346	init_waitqueue_head(&hsq->wait_queue);
347
348	return 0;
349}
350EXPORT_SYMBOL_GPL(mmc_hsq_init);
351
352void mmc_hsq_suspend(struct mmc_host *mmc)
353{
354	mmc_hsq_disable(mmc);
355}
356EXPORT_SYMBOL_GPL(mmc_hsq_suspend);
357
358int mmc_hsq_resume(struct mmc_host *mmc)
359{
360	return mmc_hsq_enable(mmc, NULL);
361}
362EXPORT_SYMBOL_GPL(mmc_hsq_resume);
363
364MODULE_DESCRIPTION("MMC Host Software Queue support");
365MODULE_LICENSE("GPL v2");