Linux Audio

Check our new training course

Loading...
v4.17
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * padata.c - generic interface to process data streams in parallel
   4 *
   5 * See Documentation/padata.txt for an api documentation.
   6 *
   7 * Copyright (C) 2008, 2009 secunet Security Networks AG
   8 * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
   9 *
  10 * This program is free software; you can redistribute it and/or modify it
  11 * under the terms and conditions of the GNU General Public License,
  12 * version 2, as published by the Free Software Foundation.
  13 *
  14 * This program is distributed in the hope it will be useful, but WITHOUT
  15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  16 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  17 * more details.
  18 *
  19 * You should have received a copy of the GNU General Public License along with
  20 * this program; if not, write to the Free Software Foundation, Inc.,
  21 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  22 */
  23
 
  24#include <linux/export.h>
  25#include <linux/cpumask.h>
  26#include <linux/err.h>
  27#include <linux/cpu.h>
  28#include <linux/padata.h>
  29#include <linux/mutex.h>
  30#include <linux/sched.h>
  31#include <linux/slab.h>
  32#include <linux/sysfs.h>
  33#include <linux/rcupdate.h>
  34#include <linux/module.h>
  35
  36#define MAX_OBJ_NUM 1000
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  37
  38static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
  39{
  40	int cpu, target_cpu;
  41
  42	target_cpu = cpumask_first(pd->cpumask.pcpu);
  43	for (cpu = 0; cpu < cpu_index; cpu++)
  44		target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu);
  45
  46	return target_cpu;
  47}
  48
  49static int padata_cpu_hash(struct parallel_data *pd)
  50{
  51	unsigned int seq_nr;
  52	int cpu_index;
  53
  54	/*
  55	 * Hash the sequence numbers to the cpus by taking
  56	 * seq_nr mod. number of cpus in use.
  57	 */
  58
  59	seq_nr = atomic_inc_return(&pd->seq_nr);
  60	cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);
  61
  62	return padata_index_to_cpu(pd, cpu_index);
  63}
  64
  65static void padata_parallel_worker(struct work_struct *parallel_work)
  66{
  67	struct padata_parallel_queue *pqueue;
  68	LIST_HEAD(local_list);
  69
  70	local_bh_disable();
  71	pqueue = container_of(parallel_work,
  72			      struct padata_parallel_queue, work);
  73
  74	spin_lock(&pqueue->parallel.lock);
  75	list_replace_init(&pqueue->parallel.list, &local_list);
  76	spin_unlock(&pqueue->parallel.lock);
  77
  78	while (!list_empty(&local_list)) {
  79		struct padata_priv *padata;
 
 
  80
  81		padata = list_entry(local_list.next,
  82				    struct padata_priv, list);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  83
  84		list_del_init(&padata->list);
 
 
 
  85
  86		padata->parallel(padata);
 
 
 
 
 
 
 
 
  87	}
 
 
 
 
  88
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  89	local_bh_enable();
  90}
  91
  92/**
  93 * padata_do_parallel - padata parallelization function
  94 *
  95 * @pinst: padata instance
  96 * @padata: object to be parallelized
  97 * @cb_cpu: cpu the serialization callback function will run on,
  98 *          must be in the serial cpumask of padata(i.e. cpumask.cbcpu).
 
 
  99 *
 100 * The parallelization callback function will run with BHs off.
 101 * Note: Every object which is parallelized by padata_do_parallel
 102 * must be seen by padata_do_serial.
 
 
 103 */
 104int padata_do_parallel(struct padata_instance *pinst,
 105		       struct padata_priv *padata, int cb_cpu)
 106{
 107	int target_cpu, err;
 108	struct padata_parallel_queue *queue;
 109	struct parallel_data *pd;
 
 110
 111	rcu_read_lock_bh();
 112
 113	pd = rcu_dereference_bh(pinst->pd);
 114
 115	err = -EINVAL;
 116	if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)
 117		goto out;
 118
 119	if (!cpumask_test_cpu(cb_cpu, pd->cpumask.cbcpu))
 120		goto out;
 
 
 
 
 
 
 
 
 121
 122	err =  -EBUSY;
 123	if ((pinst->flags & PADATA_RESET))
 124		goto out;
 125
 126	if (atomic_read(&pd->refcnt) >= MAX_OBJ_NUM)
 
 127		goto out;
 128
 129	err = 0;
 130	atomic_inc(&pd->refcnt);
 131	padata->pd = pd;
 132	padata->cb_cpu = cb_cpu;
 133
 134	target_cpu = padata_cpu_hash(pd);
 135	padata->cpu = target_cpu;
 136	queue = per_cpu_ptr(pd->pqueue, target_cpu);
 
 137
 138	spin_lock(&queue->parallel.lock);
 139	list_add_tail(&padata->list, &queue->parallel.list);
 140	spin_unlock(&queue->parallel.lock);
 
 141
 142	queue_work_on(target_cpu, pinst->wq, &queue->work);
 143
 
 
 
 
 
 
 144out:
 145	rcu_read_unlock_bh();
 146
 147	return err;
 148}
 149EXPORT_SYMBOL(padata_do_parallel);
 150
 151/*
 152 * padata_get_next - Get the next object that needs serialization.
 153 *
 154 * Return values are:
 155 *
 156 * A pointer to the control struct of the next object that needs
 157 * serialization, if present in one of the percpu reorder queues.
 158 *
 159 * -EINPROGRESS, if the next object that needs serialization will
 160 *  be parallel processed by another cpu and is not yet present in
 161 *  the cpu's reorder queue.
 162 *
 163 * -ENODATA, if this cpu has to do the parallel processing for
 164 *  the next object.
 
 
 
 
 165 */
 166static struct padata_priv *padata_get_next(struct parallel_data *pd)
 
 167{
 168	int cpu, num_cpus;
 169	unsigned int next_nr, next_index;
 170	struct padata_parallel_queue *next_queue;
 171	struct padata_priv *padata;
 172	struct padata_list *reorder;
 
 173
 174	num_cpus = cpumask_weight(pd->cpumask.pcpu);
 175
 176	/*
 177	 * Calculate the percpu reorder queue and the sequence
 178	 * number of the next object.
 179	 */
 180	next_nr = pd->processed;
 181	next_index = next_nr % num_cpus;
 182	cpu = padata_index_to_cpu(pd, next_index);
 183	next_queue = per_cpu_ptr(pd->pqueue, cpu);
 184
 185	reorder = &next_queue->reorder;
 186
 187	spin_lock(&reorder->lock);
 188	if (!list_empty(&reorder->list)) {
 189		padata = list_entry(reorder->list.next,
 190				    struct padata_priv, list);
 191
 192		list_del_init(&padata->list);
 193		atomic_dec(&pd->reorder_objects);
 194
 195		pd->processed++;
 196
 
 
 
 
 
 197		spin_unlock(&reorder->lock);
 198		goto out;
 199	}
 200	spin_unlock(&reorder->lock);
 201
 202	if (__this_cpu_read(pd->pqueue->cpu_index) == next_queue->cpu_index) {
 203		padata = ERR_PTR(-ENODATA);
 204		goto out;
 
 205	}
 206
 207	padata = ERR_PTR(-EINPROGRESS);
 208out:
 209	return padata;
 210}
 211
 212static void padata_reorder(struct parallel_data *pd)
 213{
 
 214	int cb_cpu;
 215	struct padata_priv *padata;
 216	struct padata_serial_queue *squeue;
 217	struct padata_instance *pinst = pd->pinst;
 218
 219	/*
 220	 * We need to ensure that only one cpu can work on dequeueing of
 221	 * the reorder queue the time. Calculating in which percpu reorder
 222	 * queue the next object will arrive takes some time. A spinlock
 223	 * would be highly contended. Also it is not clear in which order
 224	 * the objects arrive to the reorder queues. So a cpu could wait to
 225	 * get the lock just to notice that there is nothing to do at the
 226	 * moment. Therefore we use a trylock and let the holder of the lock
 227	 * care for all the objects enqueued during the holdtime of the lock.
 228	 */
 229	if (!spin_trylock_bh(&pd->lock))
 230		return;
 231
 232	while (1) {
 233		padata = padata_get_next(pd);
 234
 235		/*
 236		 * If the next object that needs serialization is parallel
 237		 * processed by another cpu and is still on it's way to the
 238		 * cpu's reorder queue, nothing to do for now.
 239		 */
 240		if (PTR_ERR(padata) == -EINPROGRESS)
 241			break;
 242
 243		/*
 244		 * This cpu has to do the parallel processing of the next
 245		 * object. It's waiting in the cpu's parallelization queue,
 246		 * so exit immediately.
 247		 */
 248		if (PTR_ERR(padata) == -ENODATA) {
 249			del_timer(&pd->timer);
 250			spin_unlock_bh(&pd->lock);
 251			return;
 252		}
 253
 254		cb_cpu = padata->cb_cpu;
 255		squeue = per_cpu_ptr(pd->squeue, cb_cpu);
 256
 257		spin_lock(&squeue->serial.lock);
 258		list_add_tail(&padata->list, &squeue->serial.list);
 259		spin_unlock(&squeue->serial.lock);
 260
 261		queue_work_on(cb_cpu, pinst->wq, &squeue->work);
 262	}
 263
 264	spin_unlock_bh(&pd->lock);
 265
 266	/*
 267	 * The next object that needs serialization might have arrived to
 268	 * the reorder queues in the meantime, we will be called again
 269	 * from the timer function if no one else cares for it.
 
 
 
 270	 */
 271	if (atomic_read(&pd->reorder_objects)
 272			&& !(pinst->flags & PADATA_RESET))
 273		mod_timer(&pd->timer, jiffies + HZ);
 274	else
 275		del_timer(&pd->timer);
 276
 277	return;
 
 
 
 
 
 
 
 
 278}
 279
 280static void invoke_padata_reorder(struct work_struct *work)
 281{
 282	struct padata_parallel_queue *pqueue;
 283	struct parallel_data *pd;
 284
 285	local_bh_disable();
 286	pqueue = container_of(work, struct padata_parallel_queue, reorder_work);
 287	pd = pqueue->pd;
 288	padata_reorder(pd);
 289	local_bh_enable();
 290}
 291
 292static void padata_reorder_timer(struct timer_list *t)
 293{
 294	struct parallel_data *pd = from_timer(pd, t, timer);
 295	unsigned int weight;
 296	int target_cpu, cpu;
 297
 298	cpu = get_cpu();
 299
 300	/* We don't lock pd here to not interfere with parallel processing
 301	 * padata_reorder() calls on other CPUs. We just need any CPU out of
 302	 * the cpumask.pcpu set. It would be nice if it's the right one but
 303	 * it doesn't matter if we're off to the next one by using an outdated
 304	 * pd->processed value.
 305	 */
 306	weight = cpumask_weight(pd->cpumask.pcpu);
 307	target_cpu = padata_index_to_cpu(pd, pd->processed % weight);
 308
 309	/* ensure to call the reorder callback on the correct CPU */
 310	if (cpu != target_cpu) {
 311		struct padata_parallel_queue *pqueue;
 312		struct padata_instance *pinst;
 313
 314		/* The timer function is serialized wrt itself -- no locking
 315		 * needed.
 316		 */
 317		pinst = pd->pinst;
 318		pqueue = per_cpu_ptr(pd->pqueue, target_cpu);
 319		queue_work_on(target_cpu, pinst->wq, &pqueue->reorder_work);
 320	} else {
 321		padata_reorder(pd);
 322	}
 323
 324	put_cpu();
 325}
 326
 327static void padata_serial_worker(struct work_struct *serial_work)
 328{
 329	struct padata_serial_queue *squeue;
 330	struct parallel_data *pd;
 331	LIST_HEAD(local_list);
 
 332
 333	local_bh_disable();
 334	squeue = container_of(serial_work, struct padata_serial_queue, work);
 335	pd = squeue->pd;
 336
 337	spin_lock(&squeue->serial.lock);
 338	list_replace_init(&squeue->serial.list, &local_list);
 339	spin_unlock(&squeue->serial.lock);
 340
 
 
 341	while (!list_empty(&local_list)) {
 342		struct padata_priv *padata;
 343
 344		padata = list_entry(local_list.next,
 345				    struct padata_priv, list);
 346
 347		list_del_init(&padata->list);
 348
 349		padata->serial(padata);
 350		atomic_dec(&pd->refcnt);
 351	}
 352	local_bh_enable();
 
 
 353}
 354
 355/**
 356 * padata_do_serial - padata serialization function
 357 *
 358 * @padata: object to be serialized.
 359 *
 360 * padata_do_serial must be called for every parallelized object.
 361 * The serialization callback function will run with BHs off.
 362 */
 363void padata_do_serial(struct padata_priv *padata)
 364{
 365	int cpu;
 366	struct padata_parallel_queue *pqueue;
 367	struct parallel_data *pd;
 368	int reorder_via_wq = 0;
 369
 370	pd = padata->pd;
 371
 372	cpu = get_cpu();
 
 
 
 
 
 
 
 
 
 373
 374	/* We need to run on the same CPU padata_do_parallel(.., padata, ..)
 375	 * was called on -- or, at least, enqueue the padata object into the
 376	 * correct per-cpu queue.
 
 377	 */
 378	if (cpu != padata->cpu) {
 379		reorder_via_wq = 1;
 380		cpu = padata->cpu;
 381	}
 
 382
 383	pqueue = per_cpu_ptr(pd->pqueue, cpu);
 
 
 
 384
 385	spin_lock(&pqueue->reorder.lock);
 386	atomic_inc(&pd->reorder_objects);
 387	list_add_tail(&padata->list, &pqueue->reorder.list);
 388	spin_unlock(&pqueue->reorder.lock);
 389
 390	put_cpu();
 
 
 
 391
 392	/* If we're running on the wrong CPU, call padata_reorder() via a
 393	 * kernel worker.
 394	 */
 395	if (reorder_via_wq)
 396		queue_work_on(cpu, pd->pinst->wq, &pqueue->reorder_work);
 397	else
 398		padata_reorder(pd);
 399}
 400EXPORT_SYMBOL(padata_do_serial);
 401
 402static int padata_setup_cpumasks(struct parallel_data *pd,
 403				 const struct cpumask *pcpumask,
 404				 const struct cpumask *cbcpumask)
 405{
 406	if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))
 407		return -ENOMEM;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 408
 409	cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_online_mask);
 410	if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) {
 411		free_cpumask_var(pd->cpumask.pcpu);
 412		return -ENOMEM;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 413	}
 414
 415	cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_online_mask);
 416	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 417}
 418
 419static void __padata_list_init(struct padata_list *pd_list)
 420{
 421	INIT_LIST_HEAD(&pd_list->list);
 422	spin_lock_init(&pd_list->lock);
 423}
 424
 425/* Initialize all percpu queues used by serial workers */
 426static void padata_init_squeues(struct parallel_data *pd)
 427{
 428	int cpu;
 429	struct padata_serial_queue *squeue;
 430
 431	for_each_cpu(cpu, pd->cpumask.cbcpu) {
 432		squeue = per_cpu_ptr(pd->squeue, cpu);
 433		squeue->pd = pd;
 434		__padata_list_init(&squeue->serial);
 435		INIT_WORK(&squeue->work, padata_serial_worker);
 436	}
 437}
 438
 439/* Initialize all percpu queues used by parallel workers */
 440static void padata_init_pqueues(struct parallel_data *pd)
 441{
 442	int cpu_index, cpu;
 443	struct padata_parallel_queue *pqueue;
 444
 445	cpu_index = 0;
 446	for_each_possible_cpu(cpu) {
 447		pqueue = per_cpu_ptr(pd->pqueue, cpu);
 448
 449		if (!cpumask_test_cpu(cpu, pd->cpumask.pcpu)) {
 450			pqueue->cpu_index = -1;
 451			continue;
 452		}
 453
 454		pqueue->pd = pd;
 455		pqueue->cpu_index = cpu_index;
 456		cpu_index++;
 457
 458		__padata_list_init(&pqueue->reorder);
 459		__padata_list_init(&pqueue->parallel);
 460		INIT_WORK(&pqueue->work, padata_parallel_worker);
 461		INIT_WORK(&pqueue->reorder_work, invoke_padata_reorder);
 462		atomic_set(&pqueue->num_obj, 0);
 463	}
 464}
 465
 466/* Allocate and initialize the internal cpumask dependend resources. */
 467static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,
 468					     const struct cpumask *pcpumask,
 469					     const struct cpumask *cbcpumask)
 470{
 
 471	struct parallel_data *pd;
 472
 473	pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL);
 474	if (!pd)
 475		goto err;
 476
 477	pd->pqueue = alloc_percpu(struct padata_parallel_queue);
 478	if (!pd->pqueue)
 479		goto err_free_pd;
 480
 481	pd->squeue = alloc_percpu(struct padata_serial_queue);
 482	if (!pd->squeue)
 483		goto err_free_pqueue;
 484	if (padata_setup_cpumasks(pd, pcpumask, cbcpumask) < 0)
 
 
 
 485		goto err_free_squeue;
 
 
 
 
 
 486
 487	padata_init_pqueues(pd);
 488	padata_init_squeues(pd);
 489	timer_setup(&pd->timer, padata_reorder_timer, 0);
 490	atomic_set(&pd->seq_nr, -1);
 491	atomic_set(&pd->reorder_objects, 0);
 492	atomic_set(&pd->refcnt, 0);
 493	pd->pinst = pinst;
 494	spin_lock_init(&pd->lock);
 
 
 495
 496	return pd;
 497
 
 
 498err_free_squeue:
 499	free_percpu(pd->squeue);
 500err_free_pqueue:
 501	free_percpu(pd->pqueue);
 502err_free_pd:
 503	kfree(pd);
 504err:
 505	return NULL;
 506}
 507
 508static void padata_free_pd(struct parallel_data *pd)
 509{
 510	free_cpumask_var(pd->cpumask.pcpu);
 511	free_cpumask_var(pd->cpumask.cbcpu);
 512	free_percpu(pd->pqueue);
 513	free_percpu(pd->squeue);
 514	kfree(pd);
 515}
 516
 517/* Flush all objects out of the padata queues. */
 518static void padata_flush_queues(struct parallel_data *pd)
 519{
 520	int cpu;
 521	struct padata_parallel_queue *pqueue;
 522	struct padata_serial_queue *squeue;
 523
 524	for_each_cpu(cpu, pd->cpumask.pcpu) {
 525		pqueue = per_cpu_ptr(pd->pqueue, cpu);
 526		flush_work(&pqueue->work);
 527	}
 528
 529	del_timer_sync(&pd->timer);
 530
 531	if (atomic_read(&pd->reorder_objects))
 532		padata_reorder(pd);
 533
 534	for_each_cpu(cpu, pd->cpumask.cbcpu) {
 535		squeue = per_cpu_ptr(pd->squeue, cpu);
 536		flush_work(&squeue->work);
 537	}
 538
 539	BUG_ON(atomic_read(&pd->refcnt) != 0);
 540}
 541
 542static void __padata_start(struct padata_instance *pinst)
 543{
 544	pinst->flags |= PADATA_INIT;
 545}
 546
 547static void __padata_stop(struct padata_instance *pinst)
 548{
 549	if (!(pinst->flags & PADATA_INIT))
 550		return;
 551
 552	pinst->flags &= ~PADATA_INIT;
 553
 554	synchronize_rcu();
 555
 556	get_online_cpus();
 557	padata_flush_queues(pinst->pd);
 558	put_online_cpus();
 559}
 560
 561/* Replace the internal control structure with a new one. */
 562static void padata_replace(struct padata_instance *pinst,
 563			   struct parallel_data *pd_new)
 564{
 565	struct parallel_data *pd_old = pinst->pd;
 566	int notification_mask = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 567
 568	pinst->flags |= PADATA_RESET;
 569
 570	rcu_assign_pointer(pinst->pd, pd_new);
 
 
 
 
 571
 572	synchronize_rcu();
 573
 574	if (!cpumask_equal(pd_old->cpumask.pcpu, pd_new->cpumask.pcpu))
 575		notification_mask |= PADATA_CPU_PARALLEL;
 576	if (!cpumask_equal(pd_old->cpumask.cbcpu, pd_new->cpumask.cbcpu))
 577		notification_mask |= PADATA_CPU_SERIAL;
 578
 579	padata_flush_queues(pd_old);
 580	padata_free_pd(pd_old);
 581
 582	if (notification_mask)
 583		blocking_notifier_call_chain(&pinst->cpumask_change_notifier,
 584					     notification_mask,
 585					     &pd_new->cpumask);
 586
 587	pinst->flags &= ~PADATA_RESET;
 588}
 589
 590/**
 591 * padata_register_cpumask_notifier - Registers a notifier that will be called
 592 *                             if either pcpu or cbcpu or both cpumasks change.
 593 *
 594 * @pinst: A poineter to padata instance
 595 * @nblock: A pointer to notifier block.
 596 */
 597int padata_register_cpumask_notifier(struct padata_instance *pinst,
 598				     struct notifier_block *nblock)
 599{
 600	return blocking_notifier_chain_register(&pinst->cpumask_change_notifier,
 601						nblock);
 602}
 603EXPORT_SYMBOL(padata_register_cpumask_notifier);
 604
 605/**
 606 * padata_unregister_cpumask_notifier - Unregisters cpumask notifier
 607 *        registered earlier  using padata_register_cpumask_notifier
 608 *
 609 * @pinst: A pointer to data instance.
 610 * @nlock: A pointer to notifier block.
 611 */
 612int padata_unregister_cpumask_notifier(struct padata_instance *pinst,
 613				       struct notifier_block *nblock)
 614{
 615	return blocking_notifier_chain_unregister(
 616		&pinst->cpumask_change_notifier,
 617		nblock);
 618}
 619EXPORT_SYMBOL(padata_unregister_cpumask_notifier);
 620
 621
 622/* If cpumask contains no active cpu, we mark the instance as invalid. */
 623static bool padata_validate_cpumask(struct padata_instance *pinst,
 624				    const struct cpumask *cpumask)
 625{
 626	if (!cpumask_intersects(cpumask, cpu_online_mask)) {
 627		pinst->flags |= PADATA_INVALID;
 628		return false;
 629	}
 630
 631	pinst->flags &= ~PADATA_INVALID;
 632	return true;
 633}
 634
 635static int __padata_set_cpumasks(struct padata_instance *pinst,
 636				 cpumask_var_t pcpumask,
 637				 cpumask_var_t cbcpumask)
 638{
 639	int valid;
 640	struct parallel_data *pd;
 641
 642	valid = padata_validate_cpumask(pinst, pcpumask);
 643	if (!valid) {
 644		__padata_stop(pinst);
 645		goto out_replace;
 646	}
 647
 648	valid = padata_validate_cpumask(pinst, cbcpumask);
 649	if (!valid)
 650		__padata_stop(pinst);
 651
 652out_replace:
 653	pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);
 654	if (!pd)
 655		return -ENOMEM;
 656
 657	cpumask_copy(pinst->cpumask.pcpu, pcpumask);
 658	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);
 659
 660	padata_replace(pinst, pd);
 661
 662	if (valid)
 663		__padata_start(pinst);
 664
 665	return 0;
 666}
 667
 668/**
 669 * padata_set_cpumask: Sets specified by @cpumask_type cpumask to the value
 670 *                     equivalent to @cpumask.
 671 *
 672 * @pinst: padata instance
 673 * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding
 674 *                to parallel and serial cpumasks respectively.
 675 * @cpumask: the cpumask to use
 
 
 676 */
 677int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
 678		       cpumask_var_t cpumask)
 679{
 680	struct cpumask *serial_mask, *parallel_mask;
 681	int err = -EINVAL;
 682
 
 683	mutex_lock(&pinst->lock);
 684	get_online_cpus();
 685
 686	switch (cpumask_type) {
 687	case PADATA_CPU_PARALLEL:
 688		serial_mask = pinst->cpumask.cbcpu;
 689		parallel_mask = cpumask;
 690		break;
 691	case PADATA_CPU_SERIAL:
 692		parallel_mask = pinst->cpumask.pcpu;
 693		serial_mask = cpumask;
 694		break;
 695	default:
 696		 goto out;
 697	}
 698
 699	err =  __padata_set_cpumasks(pinst, parallel_mask, serial_mask);
 700
 701out:
 702	put_online_cpus();
 703	mutex_unlock(&pinst->lock);
 
 704
 705	return err;
 706}
 707EXPORT_SYMBOL(padata_set_cpumask);
 708
 709/**
 710 * padata_start - start the parallel processing
 711 *
 712 * @pinst: padata instance to start
 713 */
 714int padata_start(struct padata_instance *pinst)
 715{
 716	int err = 0;
 717
 718	mutex_lock(&pinst->lock);
 719
 720	if (pinst->flags & PADATA_INVALID)
 721		err = -EINVAL;
 722
 723	 __padata_start(pinst);
 724
 725	mutex_unlock(&pinst->lock);
 726
 727	return err;
 728}
 729EXPORT_SYMBOL(padata_start);
 730
 731/**
 732 * padata_stop - stop the parallel processing
 733 *
 734 * @pinst: padata instance to stop
 735 */
 736void padata_stop(struct padata_instance *pinst)
 737{
 738	mutex_lock(&pinst->lock);
 739	__padata_stop(pinst);
 740	mutex_unlock(&pinst->lock);
 741}
 742EXPORT_SYMBOL(padata_stop);
 743
 744#ifdef CONFIG_HOTPLUG_CPU
 745
 746static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
 747{
 748	struct parallel_data *pd;
 749
 750	if (cpumask_test_cpu(cpu, cpu_online_mask)) {
 751		pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,
 752				     pinst->cpumask.cbcpu);
 753		if (!pd)
 754			return -ENOMEM;
 755
 756		padata_replace(pinst, pd);
 757
 758		if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&
 759		    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
 760			__padata_start(pinst);
 761	}
 762
 763	return 0;
 764}
 765
 766static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
 767{
 768	struct parallel_data *pd = NULL;
 769
 770	if (cpumask_test_cpu(cpu, cpu_online_mask)) {
 771
 
 772		if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||
 773		    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
 774			__padata_stop(pinst);
 775
 776		pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,
 777				     pinst->cpumask.cbcpu);
 778		if (!pd)
 779			return -ENOMEM;
 780
 781		padata_replace(pinst, pd);
 782
 783		cpumask_clear_cpu(cpu, pd->cpumask.cbcpu);
 784		cpumask_clear_cpu(cpu, pd->cpumask.pcpu);
 785	}
 786
 787	return 0;
 788}
 789
 790 /**
 791 * padata_remove_cpu - remove a cpu from the one or both(serial and parallel)
 792 *                     padata cpumasks.
 793 *
 794 * @pinst: padata instance
 795 * @cpu: cpu to remove
 796 * @mask: bitmask specifying from which cpumask @cpu should be removed
 797 *        The @mask may be any combination of the following flags:
 798 *          PADATA_CPU_SERIAL   - serial cpumask
 799 *          PADATA_CPU_PARALLEL - parallel cpumask
 800 */
 801int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask)
 802{
 803	int err;
 804
 805	if (!(mask & (PADATA_CPU_SERIAL | PADATA_CPU_PARALLEL)))
 806		return -EINVAL;
 807
 808	mutex_lock(&pinst->lock);
 809
 810	get_online_cpus();
 811	if (mask & PADATA_CPU_SERIAL)
 812		cpumask_clear_cpu(cpu, pinst->cpumask.cbcpu);
 813	if (mask & PADATA_CPU_PARALLEL)
 814		cpumask_clear_cpu(cpu, pinst->cpumask.pcpu);
 815
 816	err = __padata_remove_cpu(pinst, cpu);
 817	put_online_cpus();
 818
 819	mutex_unlock(&pinst->lock);
 820
 821	return err;
 822}
 823EXPORT_SYMBOL(padata_remove_cpu);
 824
 825static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)
 826{
 827	return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||
 828		cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);
 829}
 830
 831static int padata_cpu_online(unsigned int cpu, struct hlist_node *node)
 832{
 833	struct padata_instance *pinst;
 834	int ret;
 835
 836	pinst = hlist_entry_safe(node, struct padata_instance, node);
 837	if (!pinst_has_cpu(pinst, cpu))
 838		return 0;
 839
 840	mutex_lock(&pinst->lock);
 841	ret = __padata_add_cpu(pinst, cpu);
 842	mutex_unlock(&pinst->lock);
 843	return ret;
 844}
 845
 846static int padata_cpu_prep_down(unsigned int cpu, struct hlist_node *node)
 847{
 848	struct padata_instance *pinst;
 849	int ret;
 850
 851	pinst = hlist_entry_safe(node, struct padata_instance, node);
 852	if (!pinst_has_cpu(pinst, cpu))
 853		return 0;
 854
 855	mutex_lock(&pinst->lock);
 856	ret = __padata_remove_cpu(pinst, cpu);
 857	mutex_unlock(&pinst->lock);
 858	return ret;
 859}
 860
 861static enum cpuhp_state hp_online;
 862#endif
 863
 864static void __padata_free(struct padata_instance *pinst)
 865{
 866#ifdef CONFIG_HOTPLUG_CPU
 867	cpuhp_state_remove_instance_nocalls(hp_online, &pinst->node);
 
 
 868#endif
 869
 870	padata_stop(pinst);
 871	padata_free_pd(pinst->pd);
 872	free_cpumask_var(pinst->cpumask.pcpu);
 873	free_cpumask_var(pinst->cpumask.cbcpu);
 
 
 874	kfree(pinst);
 875}
 876
 877#define kobj2pinst(_kobj)					\
 878	container_of(_kobj, struct padata_instance, kobj)
 879#define attr2pentry(_attr)					\
 880	container_of(_attr, struct padata_sysfs_entry, attr)
 881
 882static void padata_sysfs_release(struct kobject *kobj)
 883{
 884	struct padata_instance *pinst = kobj2pinst(kobj);
 885	__padata_free(pinst);
 886}
 887
 888struct padata_sysfs_entry {
 889	struct attribute attr;
 890	ssize_t (*show)(struct padata_instance *, struct attribute *, char *);
 891	ssize_t (*store)(struct padata_instance *, struct attribute *,
 892			 const char *, size_t);
 893};
 894
 895static ssize_t show_cpumask(struct padata_instance *pinst,
 896			    struct attribute *attr,  char *buf)
 897{
 898	struct cpumask *cpumask;
 899	ssize_t len;
 900
 901	mutex_lock(&pinst->lock);
 902	if (!strcmp(attr->name, "serial_cpumask"))
 903		cpumask = pinst->cpumask.cbcpu;
 904	else
 905		cpumask = pinst->cpumask.pcpu;
 906
 907	len = snprintf(buf, PAGE_SIZE, "%*pb\n",
 908		       nr_cpu_ids, cpumask_bits(cpumask));
 909	mutex_unlock(&pinst->lock);
 910	return len < PAGE_SIZE ? len : -EINVAL;
 911}
 912
 913static ssize_t store_cpumask(struct padata_instance *pinst,
 914			     struct attribute *attr,
 915			     const char *buf, size_t count)
 916{
 917	cpumask_var_t new_cpumask;
 918	ssize_t ret;
 919	int mask_type;
 920
 921	if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL))
 922		return -ENOMEM;
 923
 924	ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask),
 925			   nr_cpumask_bits);
 926	if (ret < 0)
 927		goto out;
 928
 929	mask_type = !strcmp(attr->name, "serial_cpumask") ?
 930		PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL;
 931	ret = padata_set_cpumask(pinst, mask_type, new_cpumask);
 932	if (!ret)
 933		ret = count;
 934
 935out:
 936	free_cpumask_var(new_cpumask);
 937	return ret;
 938}
 939
 940#define PADATA_ATTR_RW(_name, _show_name, _store_name)		\
 941	static struct padata_sysfs_entry _name##_attr =		\
 942		__ATTR(_name, 0644, _show_name, _store_name)
 943#define PADATA_ATTR_RO(_name, _show_name)		\
 944	static struct padata_sysfs_entry _name##_attr = \
 945		__ATTR(_name, 0400, _show_name, NULL)
 946
 947PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask);
 948PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask);
 949
 950/*
 951 * Padata sysfs provides the following objects:
 952 * serial_cpumask   [RW] - cpumask for serial workers
 953 * parallel_cpumask [RW] - cpumask for parallel workers
 954 */
 955static struct attribute *padata_default_attrs[] = {
 956	&serial_cpumask_attr.attr,
 957	&parallel_cpumask_attr.attr,
 958	NULL,
 959};
 
 960
 961static ssize_t padata_sysfs_show(struct kobject *kobj,
 962				 struct attribute *attr, char *buf)
 963{
 964	struct padata_instance *pinst;
 965	struct padata_sysfs_entry *pentry;
 966	ssize_t ret = -EIO;
 967
 968	pinst = kobj2pinst(kobj);
 969	pentry = attr2pentry(attr);
 970	if (pentry->show)
 971		ret = pentry->show(pinst, attr, buf);
 972
 973	return ret;
 974}
 975
 976static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr,
 977				  const char *buf, size_t count)
 978{
 979	struct padata_instance *pinst;
 980	struct padata_sysfs_entry *pentry;
 981	ssize_t ret = -EIO;
 982
 983	pinst = kobj2pinst(kobj);
 984	pentry = attr2pentry(attr);
 985	if (pentry->show)
 986		ret = pentry->store(pinst, attr, buf, count);
 987
 988	return ret;
 989}
 990
 991static const struct sysfs_ops padata_sysfs_ops = {
 992	.show = padata_sysfs_show,
 993	.store = padata_sysfs_store,
 994};
 995
 996static struct kobj_type padata_attr_type = {
 997	.sysfs_ops = &padata_sysfs_ops,
 998	.default_attrs = padata_default_attrs,
 999	.release = padata_sysfs_release,
1000};
1001
1002/**
1003 * padata_alloc - allocate and initialize a padata instance and specify
1004 *                cpumasks for serial and parallel workers.
1005 *
1006 * @wq: workqueue to use for the allocated padata instance
1007 * @pcpumask: cpumask that will be used for padata parallelization
1008 * @cbcpumask: cpumask that will be used for padata serialization
1009 *
1010 * Must be called from a cpus_read_lock() protected region
1011 */
1012static struct padata_instance *padata_alloc(struct workqueue_struct *wq,
1013					    const struct cpumask *pcpumask,
1014					    const struct cpumask *cbcpumask)
1015{
1016	struct padata_instance *pinst;
1017	struct parallel_data *pd = NULL;
1018
1019	pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL);
1020	if (!pinst)
1021		goto err;
1022
1023	if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))
 
 
1024		goto err_free_inst;
 
 
 
 
 
 
 
 
 
 
1025	if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {
1026		free_cpumask_var(pinst->cpumask.pcpu);
1027		goto err_free_inst;
1028	}
1029	if (!padata_validate_cpumask(pinst, pcpumask) ||
1030	    !padata_validate_cpumask(pinst, cbcpumask))
1031		goto err_free_masks;
1032
1033	pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);
1034	if (!pd)
1035		goto err_free_masks;
1036
1037	rcu_assign_pointer(pinst->pd, pd);
 
1038
1039	pinst->wq = wq;
1040
1041	cpumask_copy(pinst->cpumask.pcpu, pcpumask);
1042	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);
1043
1044	pinst->flags = 0;
1045
1046	BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier);
1047	kobject_init(&pinst->kobj, &padata_attr_type);
1048	mutex_init(&pinst->lock);
1049
1050#ifdef CONFIG_HOTPLUG_CPU
1051	cpuhp_state_add_instance_nocalls_cpuslocked(hp_online, &pinst->node);
 
 
 
1052#endif
 
 
 
1053	return pinst;
1054
1055err_free_masks:
1056	free_cpumask_var(pinst->cpumask.pcpu);
1057	free_cpumask_var(pinst->cpumask.cbcpu);
 
 
 
 
 
1058err_free_inst:
1059	kfree(pinst);
1060err:
1061	return NULL;
1062}
 
1063
1064/**
1065 * padata_alloc_possible - Allocate and initialize padata instance.
1066 *                         Use the cpu_possible_mask for serial and
1067 *                         parallel workers.
1068 *
1069 * @wq: workqueue to use for the allocated padata instance
1070 *
1071 * Must be called from a cpus_read_lock() protected region
1072 */
1073struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq)
1074{
1075	lockdep_assert_cpus_held();
1076	return padata_alloc(wq, cpu_possible_mask, cpu_possible_mask);
1077}
1078EXPORT_SYMBOL(padata_alloc_possible);
1079
1080/**
1081 * padata_free - free a padata instance
 
 
1082 *
1083 * @padata_inst: padata instance to free
1084 */
1085void padata_free(struct padata_instance *pinst)
1086{
1087	kobject_put(&pinst->kobj);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1088}
1089EXPORT_SYMBOL(padata_free);
1090
1091#ifdef CONFIG_HOTPLUG_CPU
 
 
 
 
 
 
 
1092
1093static __init int padata_driver_init(void)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1094{
 
 
1095	int ret;
1096
1097	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "padata:online",
1098				      padata_cpu_online,
1099				      padata_cpu_prep_down);
1100	if (ret < 0)
1101		return ret;
1102	hp_online = ret;
1103	return 0;
1104}
1105module_init(padata_driver_init);
1106
1107static __exit void padata_driver_exit(void)
1108{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1109	cpuhp_remove_multi_state(hp_online);
1110}
1111module_exit(padata_driver_exit);
1112#endif
v6.13.7
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * padata.c - generic interface to process data streams in parallel
   4 *
   5 * See Documentation/core-api/padata.rst for more information.
   6 *
   7 * Copyright (C) 2008, 2009 secunet Security Networks AG
   8 * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
   9 *
  10 * Copyright (c) 2020 Oracle and/or its affiliates.
  11 * Author: Daniel Jordan <daniel.m.jordan@oracle.com>
 
 
 
 
 
 
 
 
 
 
  12 */
  13
  14#include <linux/completion.h>
  15#include <linux/export.h>
  16#include <linux/cpumask.h>
  17#include <linux/err.h>
  18#include <linux/cpu.h>
  19#include <linux/padata.h>
  20#include <linux/mutex.h>
  21#include <linux/sched.h>
  22#include <linux/slab.h>
  23#include <linux/sysfs.h>
  24#include <linux/rcupdate.h>
 
  25
  26#define	PADATA_WORK_ONSTACK	1	/* Work's memory is on stack */
  27
  28struct padata_work {
  29	struct work_struct	pw_work;
  30	struct list_head	pw_list;  /* padata_free_works linkage */
  31	void			*pw_data;
  32};
  33
  34static DEFINE_SPINLOCK(padata_works_lock);
  35static struct padata_work *padata_works;
  36static LIST_HEAD(padata_free_works);
  37
  38struct padata_mt_job_state {
  39	spinlock_t		lock;
  40	struct completion	completion;
  41	struct padata_mt_job	*job;
  42	int			nworks;
  43	int			nworks_fini;
  44	unsigned long		chunk_size;
  45};
  46
  47static void padata_free_pd(struct parallel_data *pd);
  48static void __init padata_mt_helper(struct work_struct *work);
  49
  50static inline void padata_get_pd(struct parallel_data *pd)
  51{
  52	refcount_inc(&pd->refcnt);
  53}
  54
  55static inline void padata_put_pd_cnt(struct parallel_data *pd, int cnt)
  56{
  57	if (refcount_sub_and_test(cnt, &pd->refcnt))
  58		padata_free_pd(pd);
  59}
  60
  61static inline void padata_put_pd(struct parallel_data *pd)
  62{
  63	padata_put_pd_cnt(pd, 1);
  64}
  65
  66static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
  67{
  68	int cpu, target_cpu;
  69
  70	target_cpu = cpumask_first(pd->cpumask.pcpu);
  71	for (cpu = 0; cpu < cpu_index; cpu++)
  72		target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu);
  73
  74	return target_cpu;
  75}
  76
  77static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr)
  78{
 
 
 
  79	/*
  80	 * Hash the sequence numbers to the cpus by taking
  81	 * seq_nr mod. number of cpus in use.
  82	 */
  83	int cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);
 
 
  84
  85	return padata_index_to_cpu(pd, cpu_index);
  86}
  87
  88static struct padata_work *padata_work_alloc(void)
  89{
  90	struct padata_work *pw;
 
  91
  92	lockdep_assert_held(&padata_works_lock);
 
 
  93
  94	if (list_empty(&padata_free_works))
  95		return NULL;	/* No more work items allowed to be queued. */
 
  96
  97	pw = list_first_entry(&padata_free_works, struct padata_work, pw_list);
  98	list_del(&pw->pw_list);
  99	return pw;
 100}
 101
 102/*
 103 * This function is marked __ref because this function may be optimized in such
 104 * a way that it directly refers to work_fn's address, which causes modpost to
 105 * complain when work_fn is marked __init. This scenario was observed with clang
 106 * LTO, where padata_work_init() was optimized to refer directly to
 107 * padata_mt_helper() because the calls to padata_work_init() with other work_fn
 108 * values were eliminated or inlined.
 109 */
 110static void __ref padata_work_init(struct padata_work *pw, work_func_t work_fn,
 111				   void *data, int flags)
 112{
 113	if (flags & PADATA_WORK_ONSTACK)
 114		INIT_WORK_ONSTACK(&pw->pw_work, work_fn);
 115	else
 116		INIT_WORK(&pw->pw_work, work_fn);
 117	pw->pw_data = data;
 118}
 119
 120static int __init padata_work_alloc_mt(int nworks, void *data,
 121				       struct list_head *head)
 122{
 123	int i;
 124
 125	spin_lock_bh(&padata_works_lock);
 126	/* Start at 1 because the current task participates in the job. */
 127	for (i = 1; i < nworks; ++i) {
 128		struct padata_work *pw = padata_work_alloc();
 129
 130		if (!pw)
 131			break;
 132		padata_work_init(pw, padata_mt_helper, data, 0);
 133		list_add(&pw->pw_list, head);
 134	}
 135	spin_unlock_bh(&padata_works_lock);
 136
 137	return i;
 138}
 139
 140static void padata_work_free(struct padata_work *pw)
 141{
 142	lockdep_assert_held(&padata_works_lock);
 143	list_add(&pw->pw_list, &padata_free_works);
 144}
 145
 146static void __init padata_works_free(struct list_head *works)
 147{
 148	struct padata_work *cur, *next;
 149
 150	if (list_empty(works))
 151		return;
 152
 153	spin_lock_bh(&padata_works_lock);
 154	list_for_each_entry_safe(cur, next, works, pw_list) {
 155		list_del(&cur->pw_list);
 156		padata_work_free(cur);
 157	}
 158	spin_unlock_bh(&padata_works_lock);
 159}
 160
 161static void padata_parallel_worker(struct work_struct *parallel_work)
 162{
 163	struct padata_work *pw = container_of(parallel_work, struct padata_work,
 164					      pw_work);
 165	struct padata_priv *padata = pw->pw_data;
 166
 167	local_bh_disable();
 168	padata->parallel(padata);
 169	spin_lock(&padata_works_lock);
 170	padata_work_free(pw);
 171	spin_unlock(&padata_works_lock);
 172	local_bh_enable();
 173}
 174
 175/**
 176 * padata_do_parallel - padata parallelization function
 177 *
 178 * @ps: padatashell
 179 * @padata: object to be parallelized
 180 * @cb_cpu: pointer to the CPU that the serialization callback function should
 181 *          run on.  If it's not in the serial cpumask of @pinst
 182 *          (i.e. cpumask.cbcpu), this function selects a fallback CPU and if
 183 *          none found, returns -EINVAL.
 184 *
 185 * The parallelization callback function will run with BHs off.
 186 * Note: Every object which is parallelized by padata_do_parallel
 187 * must be seen by padata_do_serial.
 188 *
 189 * Return: 0 on success or else negative error code.
 190 */
 191int padata_do_parallel(struct padata_shell *ps,
 192		       struct padata_priv *padata, int *cb_cpu)
 193{
 194	struct padata_instance *pinst = ps->pinst;
 195	int i, cpu, cpu_index, err;
 196	struct parallel_data *pd;
 197	struct padata_work *pw;
 198
 199	rcu_read_lock_bh();
 200
 201	pd = rcu_dereference_bh(ps->pd);
 202
 203	err = -EINVAL;
 204	if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)
 205		goto out;
 206
 207	if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) {
 208		if (cpumask_empty(pd->cpumask.cbcpu))
 209			goto out;
 210
 211		/* Select an alternate fallback CPU and notify the caller. */
 212		cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu);
 213
 214		cpu = cpumask_first(pd->cpumask.cbcpu);
 215		for (i = 0; i < cpu_index; i++)
 216			cpu = cpumask_next(cpu, pd->cpumask.cbcpu);
 217
 218		*cb_cpu = cpu;
 219	}
 
 220
 221	err = -EBUSY;
 222	if ((pinst->flags & PADATA_RESET))
 223		goto out;
 224
 225	padata_get_pd(pd);
 
 226	padata->pd = pd;
 227	padata->cb_cpu = *cb_cpu;
 228
 229	spin_lock(&padata_works_lock);
 230	padata->seq_nr = ++pd->seq_nr;
 231	pw = padata_work_alloc();
 232	spin_unlock(&padata_works_lock);
 233
 234	if (!pw) {
 235		/* Maximum works limit exceeded, run in the current task. */
 236		padata->parallel(padata);
 237	}
 238
 239	rcu_read_unlock_bh();
 240
 241	if (pw) {
 242		padata_work_init(pw, padata_parallel_worker, padata, 0);
 243		queue_work(pinst->parallel_wq, &pw->pw_work);
 244	}
 245
 246	return 0;
 247out:
 248	rcu_read_unlock_bh();
 249
 250	return err;
 251}
 252EXPORT_SYMBOL(padata_do_parallel);
 253
 254/*
 255 * padata_find_next - Find the next object that needs serialization.
 
 
 
 
 
 
 
 
 
 256 *
 257 * Return:
 258 * * A pointer to the control struct of the next object that needs
 259 *   serialization, if present in one of the percpu reorder queues.
 260 * * NULL, if the next object that needs serialization will
 261 *   be parallel processed by another cpu and is not yet present in
 262 *   the cpu's reorder queue.
 263 */
 264static struct padata_priv *padata_find_next(struct parallel_data *pd,
 265					    bool remove_object)
 266{
 
 
 
 267	struct padata_priv *padata;
 268	struct padata_list *reorder;
 269	int cpu = pd->cpu;
 270
 271	reorder = per_cpu_ptr(pd->reorder_list, cpu);
 
 
 
 
 
 
 
 
 
 
 
 272
 273	spin_lock(&reorder->lock);
 274	if (list_empty(&reorder->list)) {
 275		spin_unlock(&reorder->lock);
 276		return NULL;
 277	}
 
 
 278
 279	padata = list_entry(reorder->list.next, struct padata_priv, list);
 280
 281	/*
 282	 * Checks the rare case where two or more parallel jobs have hashed to
 283	 * the same CPU and one of the later ones finishes first.
 284	 */
 285	if (padata->seq_nr != pd->processed) {
 286		spin_unlock(&reorder->lock);
 287		return NULL;
 288	}
 
 289
 290	if (remove_object) {
 291		list_del_init(&padata->list);
 292		++pd->processed;
 293		pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false);
 294	}
 295
 296	spin_unlock(&reorder->lock);
 
 297	return padata;
 298}
 299
 300static void padata_reorder(struct parallel_data *pd)
 301{
 302	struct padata_instance *pinst = pd->ps->pinst;
 303	int cb_cpu;
 304	struct padata_priv *padata;
 305	struct padata_serial_queue *squeue;
 306	struct padata_list *reorder;
 307
 308	/*
 309	 * We need to ensure that only one cpu can work on dequeueing of
 310	 * the reorder queue the time. Calculating in which percpu reorder
 311	 * queue the next object will arrive takes some time. A spinlock
 312	 * would be highly contended. Also it is not clear in which order
 313	 * the objects arrive to the reorder queues. So a cpu could wait to
 314	 * get the lock just to notice that there is nothing to do at the
 315	 * moment. Therefore we use a trylock and let the holder of the lock
 316	 * care for all the objects enqueued during the holdtime of the lock.
 317	 */
 318	if (!spin_trylock_bh(&pd->lock))
 319		return;
 320
 321	while (1) {
 322		padata = padata_find_next(pd, true);
 323
 324		/*
 325		 * If the next object that needs serialization is parallel
 326		 * processed by another cpu and is still on it's way to the
 327		 * cpu's reorder queue, nothing to do for now.
 328		 */
 329		if (!padata)
 330			break;
 331
 
 
 
 
 
 
 
 
 
 
 
 332		cb_cpu = padata->cb_cpu;
 333		squeue = per_cpu_ptr(pd->squeue, cb_cpu);
 334
 335		spin_lock(&squeue->serial.lock);
 336		list_add_tail(&padata->list, &squeue->serial.list);
 337		spin_unlock(&squeue->serial.lock);
 338
 339		queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work);
 340	}
 341
 342	spin_unlock_bh(&pd->lock);
 343
 344	/*
 345	 * The next object that needs serialization might have arrived to
 346	 * the reorder queues in the meantime.
 347	 *
 348	 * Ensure reorder queue is read after pd->lock is dropped so we see
 349	 * new objects from another task in padata_do_serial.  Pairs with
 350	 * smp_mb in padata_do_serial.
 351	 */
 352	smp_mb();
 
 
 
 
 353
 354	reorder = per_cpu_ptr(pd->reorder_list, pd->cpu);
 355	if (!list_empty(&reorder->list) && padata_find_next(pd, false)) {
 356		/*
 357		 * Other context(eg. the padata_serial_worker) can finish the request.
 358		 * To avoid UAF issue, add pd ref here, and put pd ref after reorder_work finish.
 359		 */
 360		padata_get_pd(pd);
 361		queue_work(pinst->serial_wq, &pd->reorder_work);
 362	}
 363}
 364
 365static void invoke_padata_reorder(struct work_struct *work)
 366{
 
 367	struct parallel_data *pd;
 368
 369	local_bh_disable();
 370	pd = container_of(work, struct parallel_data, reorder_work);
 
 371	padata_reorder(pd);
 372	local_bh_enable();
 373	/* Pairs with putting the reorder_work in the serial_wq */
 374	padata_put_pd(pd);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 375}
 376
 377static void padata_serial_worker(struct work_struct *serial_work)
 378{
 379	struct padata_serial_queue *squeue;
 380	struct parallel_data *pd;
 381	LIST_HEAD(local_list);
 382	int cnt;
 383
 384	local_bh_disable();
 385	squeue = container_of(serial_work, struct padata_serial_queue, work);
 386	pd = squeue->pd;
 387
 388	spin_lock(&squeue->serial.lock);
 389	list_replace_init(&squeue->serial.list, &local_list);
 390	spin_unlock(&squeue->serial.lock);
 391
 392	cnt = 0;
 393
 394	while (!list_empty(&local_list)) {
 395		struct padata_priv *padata;
 396
 397		padata = list_entry(local_list.next,
 398				    struct padata_priv, list);
 399
 400		list_del_init(&padata->list);
 401
 402		padata->serial(padata);
 403		cnt++;
 404	}
 405	local_bh_enable();
 406
 407	padata_put_pd_cnt(pd, cnt);
 408}
 409
 410/**
 411 * padata_do_serial - padata serialization function
 412 *
 413 * @padata: object to be serialized.
 414 *
 415 * padata_do_serial must be called for every parallelized object.
 416 * The serialization callback function will run with BHs off.
 417 */
 418void padata_do_serial(struct padata_priv *padata)
 419{
 420	struct parallel_data *pd = padata->pd;
 421	int hashed_cpu = padata_cpu_hash(pd, padata->seq_nr);
 422	struct padata_list *reorder = per_cpu_ptr(pd->reorder_list, hashed_cpu);
 423	struct padata_priv *cur;
 424	struct list_head *pos;
 
 425
 426	spin_lock(&reorder->lock);
 427	/* Sort in ascending order of sequence number. */
 428	list_for_each_prev(pos, &reorder->list) {
 429		cur = list_entry(pos, struct padata_priv, list);
 430		/* Compare by difference to consider integer wrap around */
 431		if ((signed int)(cur->seq_nr - padata->seq_nr) < 0)
 432			break;
 433	}
 434	list_add(&padata->list, pos);
 435	spin_unlock(&reorder->lock);
 436
 437	/*
 438	 * Ensure the addition to the reorder list is ordered correctly
 439	 * with the trylock of pd->lock in padata_reorder.  Pairs with smp_mb
 440	 * in padata_reorder.
 441	 */
 442	smp_mb();
 443
 444	padata_reorder(pd);
 445}
 446EXPORT_SYMBOL(padata_do_serial);
 447
 448static int padata_setup_cpumasks(struct padata_instance *pinst)
 449{
 450	struct workqueue_attrs *attrs;
 451	int err;
 452
 453	attrs = alloc_workqueue_attrs();
 454	if (!attrs)
 455		return -ENOMEM;
 
 456
 457	/* Restrict parallel_wq workers to pd->cpumask.pcpu. */
 458	cpumask_copy(attrs->cpumask, pinst->cpumask.pcpu);
 459	err = apply_workqueue_attrs(pinst->parallel_wq, attrs);
 460	free_workqueue_attrs(attrs);
 461
 462	return err;
 
 
 
 
 
 
 463}
 
 464
 465static void __init padata_mt_helper(struct work_struct *w)
 
 
 466{
 467	struct padata_work *pw = container_of(w, struct padata_work, pw_work);
 468	struct padata_mt_job_state *ps = pw->pw_data;
 469	struct padata_mt_job *job = ps->job;
 470	bool done;
 471
 472	spin_lock(&ps->lock);
 473
 474	while (job->size > 0) {
 475		unsigned long start, size, end;
 476
 477		start = job->start;
 478		/* So end is chunk size aligned if enough work remains. */
 479		size = roundup(start + 1, ps->chunk_size) - start;
 480		size = min(size, job->size);
 481		end = start + size;
 482
 483		job->start = end;
 484		job->size -= size;
 485
 486		spin_unlock(&ps->lock);
 487		job->thread_fn(start, end, job->fn_arg);
 488		spin_lock(&ps->lock);
 489	}
 490
 491	++ps->nworks_fini;
 492	done = (ps->nworks_fini == ps->nworks);
 493	spin_unlock(&ps->lock);
 494
 495	if (done)
 496		complete(&ps->completion);
 497}
 498
 499/**
 500 * padata_do_multithreaded - run a multithreaded job
 501 * @job: Description of the job.
 502 *
 503 * See the definition of struct padata_mt_job for more details.
 504 */
 505void __init padata_do_multithreaded(struct padata_mt_job *job)
 506{
 507	/* In case threads finish at different times. */
 508	static const unsigned long load_balance_factor = 4;
 509	struct padata_work my_work, *pw;
 510	struct padata_mt_job_state ps;
 511	LIST_HEAD(works);
 512	int nworks, nid;
 513	static atomic_t last_used_nid __initdata;
 514
 515	if (job->size == 0)
 516		return;
 517
 518	/* Ensure at least one thread when size < min_chunk. */
 519	nworks = max(job->size / max(job->min_chunk, job->align), 1ul);
 520	nworks = min(nworks, job->max_threads);
 521
 522	if (nworks == 1) {
 523		/* Single thread, no coordination needed, cut to the chase. */
 524		job->thread_fn(job->start, job->start + job->size, job->fn_arg);
 525		return;
 526	}
 527
 528	spin_lock_init(&ps.lock);
 529	init_completion(&ps.completion);
 530	ps.job	       = job;
 531	ps.nworks      = padata_work_alloc_mt(nworks, &ps, &works);
 532	ps.nworks_fini = 0;
 533
 534	/*
 535	 * Chunk size is the amount of work a helper does per call to the
 536	 * thread function.  Load balance large jobs between threads by
 537	 * increasing the number of chunks, guarantee at least the minimum
 538	 * chunk size from the caller, and honor the caller's alignment.
 539	 * Ensure chunk_size is at least 1 to prevent divide-by-0
 540	 * panic in padata_mt_helper().
 541	 */
 542	ps.chunk_size = job->size / (ps.nworks * load_balance_factor);
 543	ps.chunk_size = max(ps.chunk_size, job->min_chunk);
 544	ps.chunk_size = max(ps.chunk_size, 1ul);
 545	ps.chunk_size = roundup(ps.chunk_size, job->align);
 546
 547	list_for_each_entry(pw, &works, pw_list)
 548		if (job->numa_aware) {
 549			int old_node = atomic_read(&last_used_nid);
 550
 551			do {
 552				nid = next_node_in(old_node, node_states[N_CPU]);
 553			} while (!atomic_try_cmpxchg(&last_used_nid, &old_node, nid));
 554			queue_work_node(nid, system_unbound_wq, &pw->pw_work);
 555		} else {
 556			queue_work(system_unbound_wq, &pw->pw_work);
 557		}
 558
 559	/* Use the current thread, which saves starting a workqueue worker. */
 560	padata_work_init(&my_work, padata_mt_helper, &ps, PADATA_WORK_ONSTACK);
 561	padata_mt_helper(&my_work.pw_work);
 562
 563	/* Wait for all the helpers to finish. */
 564	wait_for_completion(&ps.completion);
 565
 566	destroy_work_on_stack(&my_work.pw_work);
 567	padata_works_free(&works);
 568}
 569
 570static void __padata_list_init(struct padata_list *pd_list)
 571{
 572	INIT_LIST_HEAD(&pd_list->list);
 573	spin_lock_init(&pd_list->lock);
 574}
 575
 576/* Initialize all percpu queues used by serial workers */
 577static void padata_init_squeues(struct parallel_data *pd)
 578{
 579	int cpu;
 580	struct padata_serial_queue *squeue;
 581
 582	for_each_cpu(cpu, pd->cpumask.cbcpu) {
 583		squeue = per_cpu_ptr(pd->squeue, cpu);
 584		squeue->pd = pd;
 585		__padata_list_init(&squeue->serial);
 586		INIT_WORK(&squeue->work, padata_serial_worker);
 587	}
 588}
 589
 590/* Initialize per-CPU reorder lists */
 591static void padata_init_reorder_list(struct parallel_data *pd)
 592{
 593	int cpu;
 594	struct padata_list *list;
 
 
 
 
 
 
 
 
 
 595
 596	for_each_cpu(cpu, pd->cpumask.pcpu) {
 597		list = per_cpu_ptr(pd->reorder_list, cpu);
 598		__padata_list_init(list);
 
 
 
 
 
 
 599	}
 600}
 601
 602/* Allocate and initialize the internal cpumask dependend resources. */
 603static struct parallel_data *padata_alloc_pd(struct padata_shell *ps)
 
 
 604{
 605	struct padata_instance *pinst = ps->pinst;
 606	struct parallel_data *pd;
 607
 608	pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL);
 609	if (!pd)
 610		goto err;
 611
 612	pd->reorder_list = alloc_percpu(struct padata_list);
 613	if (!pd->reorder_list)
 614		goto err_free_pd;
 615
 616	pd->squeue = alloc_percpu(struct padata_serial_queue);
 617	if (!pd->squeue)
 618		goto err_free_reorder_list;
 619
 620	pd->ps = ps;
 621
 622	if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))
 623		goto err_free_squeue;
 624	if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL))
 625		goto err_free_pcpu;
 626
 627	cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask);
 628	cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask);
 629
 630	padata_init_reorder_list(pd);
 631	padata_init_squeues(pd);
 632	pd->seq_nr = -1;
 633	refcount_set(&pd->refcnt, 1);
 
 
 
 634	spin_lock_init(&pd->lock);
 635	pd->cpu = cpumask_first(pd->cpumask.pcpu);
 636	INIT_WORK(&pd->reorder_work, invoke_padata_reorder);
 637
 638	return pd;
 639
 640err_free_pcpu:
 641	free_cpumask_var(pd->cpumask.pcpu);
 642err_free_squeue:
 643	free_percpu(pd->squeue);
 644err_free_reorder_list:
 645	free_percpu(pd->reorder_list);
 646err_free_pd:
 647	kfree(pd);
 648err:
 649	return NULL;
 650}
 651
 652static void padata_free_pd(struct parallel_data *pd)
 653{
 654	free_cpumask_var(pd->cpumask.pcpu);
 655	free_cpumask_var(pd->cpumask.cbcpu);
 656	free_percpu(pd->reorder_list);
 657	free_percpu(pd->squeue);
 658	kfree(pd);
 659}
 660
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 661static void __padata_start(struct padata_instance *pinst)
 662{
 663	pinst->flags |= PADATA_INIT;
 664}
 665
 666static void __padata_stop(struct padata_instance *pinst)
 667{
 668	if (!(pinst->flags & PADATA_INIT))
 669		return;
 670
 671	pinst->flags &= ~PADATA_INIT;
 672
 673	synchronize_rcu();
 
 
 
 
 674}
 675
 676/* Replace the internal control structure with a new one. */
 677static int padata_replace_one(struct padata_shell *ps)
 
 678{
 679	struct parallel_data *pd_new;
 680
 681	pd_new = padata_alloc_pd(ps);
 682	if (!pd_new)
 683		return -ENOMEM;
 684
 685	ps->opd = rcu_dereference_protected(ps->pd, 1);
 686	rcu_assign_pointer(ps->pd, pd_new);
 687
 688	return 0;
 689}
 690
 691static int padata_replace(struct padata_instance *pinst)
 692{
 693	struct padata_shell *ps;
 694	int err = 0;
 695
 696	pinst->flags |= PADATA_RESET;
 697
 698	list_for_each_entry(ps, &pinst->pslist, list) {
 699		err = padata_replace_one(ps);
 700		if (err)
 701			break;
 702	}
 703
 704	synchronize_rcu();
 705
 706	list_for_each_entry_continue_reverse(ps, &pinst->pslist, list)
 707		padata_put_pd(ps->opd);
 
 
 
 
 
 
 
 
 
 
 708
 709	pinst->flags &= ~PADATA_RESET;
 
 710
 711	return err;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 712}
 
 
 713
 714/* If cpumask contains no active cpu, we mark the instance as invalid. */
 715static bool padata_validate_cpumask(struct padata_instance *pinst,
 716				    const struct cpumask *cpumask)
 717{
 718	if (!cpumask_intersects(cpumask, cpu_online_mask)) {
 719		pinst->flags |= PADATA_INVALID;
 720		return false;
 721	}
 722
 723	pinst->flags &= ~PADATA_INVALID;
 724	return true;
 725}
 726
 727static int __padata_set_cpumasks(struct padata_instance *pinst,
 728				 cpumask_var_t pcpumask,
 729				 cpumask_var_t cbcpumask)
 730{
 731	int valid;
 732	int err;
 733
 734	valid = padata_validate_cpumask(pinst, pcpumask);
 735	if (!valid) {
 736		__padata_stop(pinst);
 737		goto out_replace;
 738	}
 739
 740	valid = padata_validate_cpumask(pinst, cbcpumask);
 741	if (!valid)
 742		__padata_stop(pinst);
 743
 744out_replace:
 
 
 
 
 745	cpumask_copy(pinst->cpumask.pcpu, pcpumask);
 746	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);
 747
 748	err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst);
 749
 750	if (valid)
 751		__padata_start(pinst);
 752
 753	return err;
 754}
 755
 756/**
 757 * padata_set_cpumask - Sets specified by @cpumask_type cpumask to the value
 758 *                      equivalent to @cpumask.
 
 759 * @pinst: padata instance
 760 * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding
 761 *                to parallel and serial cpumasks respectively.
 762 * @cpumask: the cpumask to use
 763 *
 764 * Return: 0 on success or negative error code
 765 */
 766int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
 767		       cpumask_var_t cpumask)
 768{
 769	struct cpumask *serial_mask, *parallel_mask;
 770	int err = -EINVAL;
 771
 772	cpus_read_lock();
 773	mutex_lock(&pinst->lock);
 
 774
 775	switch (cpumask_type) {
 776	case PADATA_CPU_PARALLEL:
 777		serial_mask = pinst->cpumask.cbcpu;
 778		parallel_mask = cpumask;
 779		break;
 780	case PADATA_CPU_SERIAL:
 781		parallel_mask = pinst->cpumask.pcpu;
 782		serial_mask = cpumask;
 783		break;
 784	default:
 785		 goto out;
 786	}
 787
 788	err =  __padata_set_cpumasks(pinst, parallel_mask, serial_mask);
 789
 790out:
 
 791	mutex_unlock(&pinst->lock);
 792	cpus_read_unlock();
 793
 794	return err;
 795}
 796EXPORT_SYMBOL(padata_set_cpumask);
 797
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 798#ifdef CONFIG_HOTPLUG_CPU
 799
 800static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
 801{
 802	int err = 0;
 803
 804	if (cpumask_test_cpu(cpu, cpu_online_mask)) {
 805		err = padata_replace(pinst);
 
 
 
 
 
 806
 807		if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&
 808		    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
 809			__padata_start(pinst);
 810	}
 811
 812	return err;
 813}
 814
 815static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
 816{
 817	int err = 0;
 
 
 818
 819	if (!cpumask_test_cpu(cpu, cpu_online_mask)) {
 820		if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||
 821		    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
 822			__padata_stop(pinst);
 823
 824		err = padata_replace(pinst);
 
 
 
 
 
 
 
 
 825	}
 826
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 827	return err;
 828}
 
 829
 830static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)
 831{
 832	return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||
 833		cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);
 834}
 835
 836static int padata_cpu_online(unsigned int cpu, struct hlist_node *node)
 837{
 838	struct padata_instance *pinst;
 839	int ret;
 840
 841	pinst = hlist_entry_safe(node, struct padata_instance, cpu_online_node);
 842	if (!pinst_has_cpu(pinst, cpu))
 843		return 0;
 844
 845	mutex_lock(&pinst->lock);
 846	ret = __padata_add_cpu(pinst, cpu);
 847	mutex_unlock(&pinst->lock);
 848	return ret;
 849}
 850
 851static int padata_cpu_dead(unsigned int cpu, struct hlist_node *node)
 852{
 853	struct padata_instance *pinst;
 854	int ret;
 855
 856	pinst = hlist_entry_safe(node, struct padata_instance, cpu_dead_node);
 857	if (!pinst_has_cpu(pinst, cpu))
 858		return 0;
 859
 860	mutex_lock(&pinst->lock);
 861	ret = __padata_remove_cpu(pinst, cpu);
 862	mutex_unlock(&pinst->lock);
 863	return ret;
 864}
 865
 866static enum cpuhp_state hp_online;
 867#endif
 868
 869static void __padata_free(struct padata_instance *pinst)
 870{
 871#ifdef CONFIG_HOTPLUG_CPU
 872	cpuhp_state_remove_instance_nocalls(CPUHP_PADATA_DEAD,
 873					    &pinst->cpu_dead_node);
 874	cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpu_online_node);
 875#endif
 876
 877	WARN_ON(!list_empty(&pinst->pslist));
 878
 879	free_cpumask_var(pinst->cpumask.pcpu);
 880	free_cpumask_var(pinst->cpumask.cbcpu);
 881	destroy_workqueue(pinst->serial_wq);
 882	destroy_workqueue(pinst->parallel_wq);
 883	kfree(pinst);
 884}
 885
 886#define kobj2pinst(_kobj)					\
 887	container_of(_kobj, struct padata_instance, kobj)
 888#define attr2pentry(_attr)					\
 889	container_of(_attr, struct padata_sysfs_entry, attr)
 890
 891static void padata_sysfs_release(struct kobject *kobj)
 892{
 893	struct padata_instance *pinst = kobj2pinst(kobj);
 894	__padata_free(pinst);
 895}
 896
 897struct padata_sysfs_entry {
 898	struct attribute attr;
 899	ssize_t (*show)(struct padata_instance *, struct attribute *, char *);
 900	ssize_t (*store)(struct padata_instance *, struct attribute *,
 901			 const char *, size_t);
 902};
 903
 904static ssize_t show_cpumask(struct padata_instance *pinst,
 905			    struct attribute *attr,  char *buf)
 906{
 907	struct cpumask *cpumask;
 908	ssize_t len;
 909
 910	mutex_lock(&pinst->lock);
 911	if (!strcmp(attr->name, "serial_cpumask"))
 912		cpumask = pinst->cpumask.cbcpu;
 913	else
 914		cpumask = pinst->cpumask.pcpu;
 915
 916	len = snprintf(buf, PAGE_SIZE, "%*pb\n",
 917		       nr_cpu_ids, cpumask_bits(cpumask));
 918	mutex_unlock(&pinst->lock);
 919	return len < PAGE_SIZE ? len : -EINVAL;
 920}
 921
 922static ssize_t store_cpumask(struct padata_instance *pinst,
 923			     struct attribute *attr,
 924			     const char *buf, size_t count)
 925{
 926	cpumask_var_t new_cpumask;
 927	ssize_t ret;
 928	int mask_type;
 929
 930	if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL))
 931		return -ENOMEM;
 932
 933	ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask),
 934			   nr_cpumask_bits);
 935	if (ret < 0)
 936		goto out;
 937
 938	mask_type = !strcmp(attr->name, "serial_cpumask") ?
 939		PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL;
 940	ret = padata_set_cpumask(pinst, mask_type, new_cpumask);
 941	if (!ret)
 942		ret = count;
 943
 944out:
 945	free_cpumask_var(new_cpumask);
 946	return ret;
 947}
 948
 949#define PADATA_ATTR_RW(_name, _show_name, _store_name)		\
 950	static struct padata_sysfs_entry _name##_attr =		\
 951		__ATTR(_name, 0644, _show_name, _store_name)
 952#define PADATA_ATTR_RO(_name, _show_name)		\
 953	static struct padata_sysfs_entry _name##_attr = \
 954		__ATTR(_name, 0400, _show_name, NULL)
 955
 956PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask);
 957PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask);
 958
 959/*
 960 * Padata sysfs provides the following objects:
 961 * serial_cpumask   [RW] - cpumask for serial workers
 962 * parallel_cpumask [RW] - cpumask for parallel workers
 963 */
 964static struct attribute *padata_default_attrs[] = {
 965	&serial_cpumask_attr.attr,
 966	&parallel_cpumask_attr.attr,
 967	NULL,
 968};
 969ATTRIBUTE_GROUPS(padata_default);
 970
 971static ssize_t padata_sysfs_show(struct kobject *kobj,
 972				 struct attribute *attr, char *buf)
 973{
 974	struct padata_instance *pinst;
 975	struct padata_sysfs_entry *pentry;
 976	ssize_t ret = -EIO;
 977
 978	pinst = kobj2pinst(kobj);
 979	pentry = attr2pentry(attr);
 980	if (pentry->show)
 981		ret = pentry->show(pinst, attr, buf);
 982
 983	return ret;
 984}
 985
 986static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr,
 987				  const char *buf, size_t count)
 988{
 989	struct padata_instance *pinst;
 990	struct padata_sysfs_entry *pentry;
 991	ssize_t ret = -EIO;
 992
 993	pinst = kobj2pinst(kobj);
 994	pentry = attr2pentry(attr);
 995	if (pentry->store)
 996		ret = pentry->store(pinst, attr, buf, count);
 997
 998	return ret;
 999}
1000
1001static const struct sysfs_ops padata_sysfs_ops = {
1002	.show = padata_sysfs_show,
1003	.store = padata_sysfs_store,
1004};
1005
1006static const struct kobj_type padata_attr_type = {
1007	.sysfs_ops = &padata_sysfs_ops,
1008	.default_groups = padata_default_groups,
1009	.release = padata_sysfs_release,
1010};
1011
1012/**
1013 * padata_alloc - allocate and initialize a padata instance
1014 * @name: used to identify the instance
1015 *
1016 * Return: new instance on success, NULL on error
 
 
 
 
1017 */
1018struct padata_instance *padata_alloc(const char *name)
 
 
1019{
1020	struct padata_instance *pinst;
 
1021
1022	pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL);
1023	if (!pinst)
1024		goto err;
1025
1026	pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0,
1027					     name);
1028	if (!pinst->parallel_wq)
1029		goto err_free_inst;
1030
1031	cpus_read_lock();
1032
1033	pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM |
1034					   WQ_CPU_INTENSIVE, 1, name);
1035	if (!pinst->serial_wq)
1036		goto err_put_cpus;
1037
1038	if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))
1039		goto err_free_serial_wq;
1040	if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {
1041		free_cpumask_var(pinst->cpumask.pcpu);
1042		goto err_free_serial_wq;
1043	}
 
 
 
1044
1045	INIT_LIST_HEAD(&pinst->pslist);
 
 
1046
1047	cpumask_copy(pinst->cpumask.pcpu, cpu_possible_mask);
1048	cpumask_copy(pinst->cpumask.cbcpu, cpu_possible_mask);
1049
1050	if (padata_setup_cpumasks(pinst))
1051		goto err_free_masks;
 
 
1052
1053	__padata_start(pinst);
1054
 
1055	kobject_init(&pinst->kobj, &padata_attr_type);
1056	mutex_init(&pinst->lock);
1057
1058#ifdef CONFIG_HOTPLUG_CPU
1059	cpuhp_state_add_instance_nocalls_cpuslocked(hp_online,
1060						    &pinst->cpu_online_node);
1061	cpuhp_state_add_instance_nocalls_cpuslocked(CPUHP_PADATA_DEAD,
1062						    &pinst->cpu_dead_node);
1063#endif
1064
1065	cpus_read_unlock();
1066
1067	return pinst;
1068
1069err_free_masks:
1070	free_cpumask_var(pinst->cpumask.pcpu);
1071	free_cpumask_var(pinst->cpumask.cbcpu);
1072err_free_serial_wq:
1073	destroy_workqueue(pinst->serial_wq);
1074err_put_cpus:
1075	cpus_read_unlock();
1076	destroy_workqueue(pinst->parallel_wq);
1077err_free_inst:
1078	kfree(pinst);
1079err:
1080	return NULL;
1081}
1082EXPORT_SYMBOL(padata_alloc);
1083
1084/**
1085 * padata_free - free a padata instance
 
 
 
 
1086 *
1087 * @pinst: padata instance to free
1088 */
1089void padata_free(struct padata_instance *pinst)
1090{
1091	kobject_put(&pinst->kobj);
 
1092}
1093EXPORT_SYMBOL(padata_free);
1094
1095/**
1096 * padata_alloc_shell - Allocate and initialize padata shell.
1097 *
1098 * @pinst: Parent padata_instance object.
1099 *
1100 * Return: new shell on success, NULL on error
1101 */
1102struct padata_shell *padata_alloc_shell(struct padata_instance *pinst)
1103{
1104	struct parallel_data *pd;
1105	struct padata_shell *ps;
1106
1107	ps = kzalloc(sizeof(*ps), GFP_KERNEL);
1108	if (!ps)
1109		goto out;
1110
1111	ps->pinst = pinst;
1112
1113	cpus_read_lock();
1114	pd = padata_alloc_pd(ps);
1115	cpus_read_unlock();
1116
1117	if (!pd)
1118		goto out_free_ps;
1119
1120	mutex_lock(&pinst->lock);
1121	RCU_INIT_POINTER(ps->pd, pd);
1122	list_add(&ps->list, &pinst->pslist);
1123	mutex_unlock(&pinst->lock);
1124
1125	return ps;
1126
1127out_free_ps:
1128	kfree(ps);
1129out:
1130	return NULL;
1131}
1132EXPORT_SYMBOL(padata_alloc_shell);
1133
1134/**
1135 * padata_free_shell - free a padata shell
1136 *
1137 * @ps: padata shell to free
1138 */
1139void padata_free_shell(struct padata_shell *ps)
1140{
1141	struct parallel_data *pd;
1142
1143	if (!ps)
1144		return;
1145
1146	/*
1147	 * Wait for all _do_serial calls to finish to avoid touching
1148	 * freed pd's and ps's.
1149	 */
1150	synchronize_rcu();
1151
1152	mutex_lock(&ps->pinst->lock);
1153	list_del(&ps->list);
1154	pd = rcu_dereference_protected(ps->pd, 1);
1155	padata_put_pd(pd);
1156	mutex_unlock(&ps->pinst->lock);
1157
1158	kfree(ps);
1159}
1160EXPORT_SYMBOL(padata_free_shell);
1161
1162void __init padata_init(void)
1163{
1164	unsigned int i, possible_cpus;
1165#ifdef CONFIG_HOTPLUG_CPU
1166	int ret;
1167
1168	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "padata:online",
1169				      padata_cpu_online, NULL);
 
1170	if (ret < 0)
1171		goto err;
1172	hp_online = ret;
 
 
 
1173
1174	ret = cpuhp_setup_state_multi(CPUHP_PADATA_DEAD, "padata:dead",
1175				      NULL, padata_cpu_dead);
1176	if (ret < 0)
1177		goto remove_online_state;
1178#endif
1179
1180	possible_cpus = num_possible_cpus();
1181	padata_works = kmalloc_array(possible_cpus, sizeof(struct padata_work),
1182				     GFP_KERNEL);
1183	if (!padata_works)
1184		goto remove_dead_state;
1185
1186	for (i = 0; i < possible_cpus; ++i)
1187		list_add(&padata_works[i].pw_list, &padata_free_works);
1188
1189	return;
1190
1191remove_dead_state:
1192#ifdef CONFIG_HOTPLUG_CPU
1193	cpuhp_remove_multi_state(CPUHP_PADATA_DEAD);
1194remove_online_state:
1195	cpuhp_remove_multi_state(hp_online);
1196err:
 
1197#endif
1198	pr_warn("padata: initialization failed\n");
1199}