Loading...
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/*
3 * Copyright(c) 2015 - 2020 Intel Corporation.
4 */
5
6#include <linux/topology.h>
7#include <linux/cpumask.h>
8#include <linux/interrupt.h>
9#include <linux/numa.h>
10
11#include "hfi.h"
12#include "affinity.h"
13#include "sdma.h"
14#include "trace.h"
15
16struct hfi1_affinity_node_list node_affinity = {
17 .list = LIST_HEAD_INIT(node_affinity.list),
18 .lock = __MUTEX_INITIALIZER(node_affinity.lock)
19};
20
21/* Name of IRQ types, indexed by enum irq_type */
22static const char * const irq_type_names[] = {
23 "SDMA",
24 "RCVCTXT",
25 "NETDEVCTXT",
26 "GENERAL",
27 "OTHER",
28};
29
30/* Per NUMA node count of HFI devices */
31static unsigned int *hfi1_per_node_cntr;
32
33static inline void init_cpu_mask_set(struct cpu_mask_set *set)
34{
35 cpumask_clear(&set->mask);
36 cpumask_clear(&set->used);
37 set->gen = 0;
38}
39
40/* Increment generation of CPU set if needed */
41static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set)
42{
43 if (cpumask_equal(&set->mask, &set->used)) {
44 /*
45 * We've used up all the CPUs, bump up the generation
46 * and reset the 'used' map
47 */
48 set->gen++;
49 cpumask_clear(&set->used);
50 }
51}
52
53static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set)
54{
55 if (cpumask_empty(&set->used) && set->gen) {
56 set->gen--;
57 cpumask_copy(&set->used, &set->mask);
58 }
59}
60
61/* Get the first CPU from the list of unused CPUs in a CPU set data structure */
62static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff)
63{
64 int cpu;
65
66 if (!diff || !set)
67 return -EINVAL;
68
69 _cpu_mask_set_gen_inc(set);
70
71 /* Find out CPUs left in CPU mask */
72 cpumask_andnot(diff, &set->mask, &set->used);
73
74 cpu = cpumask_first(diff);
75 if (cpu >= nr_cpu_ids) /* empty */
76 cpu = -EINVAL;
77 else
78 cpumask_set_cpu(cpu, &set->used);
79
80 return cpu;
81}
82
83static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)
84{
85 if (!set)
86 return;
87
88 cpumask_clear_cpu(cpu, &set->used);
89 _cpu_mask_set_gen_dec(set);
90}
91
92/* Initialize non-HT cpu cores mask */
93void init_real_cpu_mask(void)
94{
95 int possible, curr_cpu, i, ht;
96
97 cpumask_clear(&node_affinity.real_cpu_mask);
98
99 /* Start with cpu online mask as the real cpu mask */
100 cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
101
102 /*
103 * Remove HT cores from the real cpu mask. Do this in two steps below.
104 */
105 possible = cpumask_weight(&node_affinity.real_cpu_mask);
106 ht = cpumask_weight(topology_sibling_cpumask(
107 cpumask_first(&node_affinity.real_cpu_mask)));
108 /*
109 * Step 1. Skip over the first N HT siblings and use them as the
110 * "real" cores. Assumes that HT cores are not enumerated in
111 * succession (except in the single core case).
112 */
113 curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
114 for (i = 0; i < possible / ht; i++)
115 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
116 /*
117 * Step 2. Remove the remaining HT siblings. Use cpumask_next() to
118 * skip any gaps.
119 */
120 for (; i < possible; i++) {
121 cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
122 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
123 }
124}
125
126int node_affinity_init(void)
127{
128 int node;
129 struct pci_dev *dev = NULL;
130 const struct pci_device_id *ids = hfi1_pci_tbl;
131
132 cpumask_clear(&node_affinity.proc.used);
133 cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
134
135 node_affinity.proc.gen = 0;
136 node_affinity.num_core_siblings =
137 cpumask_weight(topology_sibling_cpumask(
138 cpumask_first(&node_affinity.proc.mask)
139 ));
140 node_affinity.num_possible_nodes = num_possible_nodes();
141 node_affinity.num_online_nodes = num_online_nodes();
142 node_affinity.num_online_cpus = num_online_cpus();
143
144 /*
145 * The real cpu mask is part of the affinity struct but it has to be
146 * initialized early. It is needed to calculate the number of user
147 * contexts in set_up_context_variables().
148 */
149 init_real_cpu_mask();
150
151 hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes,
152 sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
153 if (!hfi1_per_node_cntr)
154 return -ENOMEM;
155
156 while (ids->vendor) {
157 dev = NULL;
158 while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
159 node = pcibus_to_node(dev->bus);
160 if (node < 0)
161 goto out;
162
163 hfi1_per_node_cntr[node]++;
164 }
165 ids++;
166 }
167
168 return 0;
169
170out:
171 /*
172 * Invalid PCI NUMA node information found, note it, and populate
173 * our database 1:1.
174 */
175 pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n");
176 pr_err("HFI: System BIOS may need to be upgraded\n");
177 for (node = 0; node < node_affinity.num_possible_nodes; node++)
178 hfi1_per_node_cntr[node] = 1;
179
180 pci_dev_put(dev);
181
182 return 0;
183}
184
185static void node_affinity_destroy(struct hfi1_affinity_node *entry)
186{
187 free_percpu(entry->comp_vect_affinity);
188 kfree(entry);
189}
190
191void node_affinity_destroy_all(void)
192{
193 struct list_head *pos, *q;
194 struct hfi1_affinity_node *entry;
195
196 mutex_lock(&node_affinity.lock);
197 list_for_each_safe(pos, q, &node_affinity.list) {
198 entry = list_entry(pos, struct hfi1_affinity_node,
199 list);
200 list_del(pos);
201 node_affinity_destroy(entry);
202 }
203 mutex_unlock(&node_affinity.lock);
204 kfree(hfi1_per_node_cntr);
205}
206
207static struct hfi1_affinity_node *node_affinity_allocate(int node)
208{
209 struct hfi1_affinity_node *entry;
210
211 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
212 if (!entry)
213 return NULL;
214 entry->node = node;
215 entry->comp_vect_affinity = alloc_percpu(u16);
216 INIT_LIST_HEAD(&entry->list);
217
218 return entry;
219}
220
221/*
222 * It appends an entry to the list.
223 * It *must* be called with node_affinity.lock held.
224 */
225static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
226{
227 list_add_tail(&entry->list, &node_affinity.list);
228}
229
230/* It must be called with node_affinity.lock held */
231static struct hfi1_affinity_node *node_affinity_lookup(int node)
232{
233 struct hfi1_affinity_node *entry;
234
235 list_for_each_entry(entry, &node_affinity.list, list) {
236 if (entry->node == node)
237 return entry;
238 }
239
240 return NULL;
241}
242
243static int per_cpu_affinity_get(cpumask_var_t possible_cpumask,
244 u16 __percpu *comp_vect_affinity)
245{
246 int curr_cpu;
247 u16 cntr;
248 u16 prev_cntr;
249 int ret_cpu;
250
251 if (!possible_cpumask) {
252 ret_cpu = -EINVAL;
253 goto fail;
254 }
255
256 if (!comp_vect_affinity) {
257 ret_cpu = -EINVAL;
258 goto fail;
259 }
260
261 ret_cpu = cpumask_first(possible_cpumask);
262 if (ret_cpu >= nr_cpu_ids) {
263 ret_cpu = -EINVAL;
264 goto fail;
265 }
266
267 prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu);
268 for_each_cpu(curr_cpu, possible_cpumask) {
269 cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
270
271 if (cntr < prev_cntr) {
272 ret_cpu = curr_cpu;
273 prev_cntr = cntr;
274 }
275 }
276
277 *per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1;
278
279fail:
280 return ret_cpu;
281}
282
283static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask,
284 u16 __percpu *comp_vect_affinity)
285{
286 int curr_cpu;
287 int max_cpu;
288 u16 cntr;
289 u16 prev_cntr;
290
291 if (!possible_cpumask)
292 return -EINVAL;
293
294 if (!comp_vect_affinity)
295 return -EINVAL;
296
297 max_cpu = cpumask_first(possible_cpumask);
298 if (max_cpu >= nr_cpu_ids)
299 return -EINVAL;
300
301 prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu);
302 for_each_cpu(curr_cpu, possible_cpumask) {
303 cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
304
305 if (cntr > prev_cntr) {
306 max_cpu = curr_cpu;
307 prev_cntr = cntr;
308 }
309 }
310
311 *per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1;
312
313 return max_cpu;
314}
315
316/*
317 * Non-interrupt CPUs are used first, then interrupt CPUs.
318 * Two already allocated cpu masks must be passed.
319 */
320static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,
321 struct hfi1_affinity_node *entry,
322 cpumask_var_t non_intr_cpus,
323 cpumask_var_t available_cpus)
324 __must_hold(&node_affinity.lock)
325{
326 int cpu;
327 struct cpu_mask_set *set = dd->comp_vect;
328
329 lockdep_assert_held(&node_affinity.lock);
330 if (!non_intr_cpus) {
331 cpu = -1;
332 goto fail;
333 }
334
335 if (!available_cpus) {
336 cpu = -1;
337 goto fail;
338 }
339
340 /* Available CPUs for pinning completion vectors */
341 _cpu_mask_set_gen_inc(set);
342 cpumask_andnot(available_cpus, &set->mask, &set->used);
343
344 /* Available CPUs without SDMA engine interrupts */
345 cpumask_andnot(non_intr_cpus, available_cpus,
346 &entry->def_intr.used);
347
348 /* If there are non-interrupt CPUs available, use them first */
349 if (!cpumask_empty(non_intr_cpus))
350 cpu = cpumask_first(non_intr_cpus);
351 else /* Otherwise, use interrupt CPUs */
352 cpu = cpumask_first(available_cpus);
353
354 if (cpu >= nr_cpu_ids) { /* empty */
355 cpu = -1;
356 goto fail;
357 }
358 cpumask_set_cpu(cpu, &set->used);
359
360fail:
361 return cpu;
362}
363
364static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu)
365{
366 struct cpu_mask_set *set = dd->comp_vect;
367
368 if (cpu < 0)
369 return;
370
371 cpu_mask_set_put(set, cpu);
372}
373
374/* _dev_comp_vect_mappings_destroy() is reentrant */
375static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
376{
377 int i, cpu;
378
379 if (!dd->comp_vect_mappings)
380 return;
381
382 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
383 cpu = dd->comp_vect_mappings[i];
384 _dev_comp_vect_cpu_put(dd, cpu);
385 dd->comp_vect_mappings[i] = -1;
386 hfi1_cdbg(AFFINITY,
387 "[%s] Release CPU %d from completion vector %d",
388 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i);
389 }
390
391 kfree(dd->comp_vect_mappings);
392 dd->comp_vect_mappings = NULL;
393}
394
395/*
396 * This function creates the table for looking up CPUs for completion vectors.
397 * num_comp_vectors needs to have been initilized before calling this function.
398 */
399static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
400 struct hfi1_affinity_node *entry)
401 __must_hold(&node_affinity.lock)
402{
403 int i, cpu, ret;
404 cpumask_var_t non_intr_cpus;
405 cpumask_var_t available_cpus;
406
407 lockdep_assert_held(&node_affinity.lock);
408
409 if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL))
410 return -ENOMEM;
411
412 if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) {
413 free_cpumask_var(non_intr_cpus);
414 return -ENOMEM;
415 }
416
417 dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus,
418 sizeof(*dd->comp_vect_mappings),
419 GFP_KERNEL);
420 if (!dd->comp_vect_mappings) {
421 ret = -ENOMEM;
422 goto fail;
423 }
424 for (i = 0; i < dd->comp_vect_possible_cpus; i++)
425 dd->comp_vect_mappings[i] = -1;
426
427 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
428 cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus,
429 available_cpus);
430 if (cpu < 0) {
431 ret = -EINVAL;
432 goto fail;
433 }
434
435 dd->comp_vect_mappings[i] = cpu;
436 hfi1_cdbg(AFFINITY,
437 "[%s] Completion Vector %d -> CPU %d",
438 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu);
439 }
440
441 free_cpumask_var(available_cpus);
442 free_cpumask_var(non_intr_cpus);
443 return 0;
444
445fail:
446 free_cpumask_var(available_cpus);
447 free_cpumask_var(non_intr_cpus);
448 _dev_comp_vect_mappings_destroy(dd);
449
450 return ret;
451}
452
453int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd)
454{
455 int ret;
456 struct hfi1_affinity_node *entry;
457
458 mutex_lock(&node_affinity.lock);
459 entry = node_affinity_lookup(dd->node);
460 if (!entry) {
461 ret = -EINVAL;
462 goto unlock;
463 }
464 ret = _dev_comp_vect_mappings_create(dd, entry);
465unlock:
466 mutex_unlock(&node_affinity.lock);
467
468 return ret;
469}
470
471void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd)
472{
473 _dev_comp_vect_mappings_destroy(dd);
474}
475
476int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect)
477{
478 struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
479 struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
480
481 if (!dd->comp_vect_mappings)
482 return -EINVAL;
483 if (comp_vect >= dd->comp_vect_possible_cpus)
484 return -EINVAL;
485
486 return dd->comp_vect_mappings[comp_vect];
487}
488
489/*
490 * It assumes dd->comp_vect_possible_cpus is available.
491 */
492static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd,
493 struct hfi1_affinity_node *entry,
494 bool first_dev_init)
495 __must_hold(&node_affinity.lock)
496{
497 int i, j, curr_cpu;
498 int possible_cpus_comp_vect = 0;
499 struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask;
500
501 lockdep_assert_held(&node_affinity.lock);
502 /*
503 * If there's only one CPU available for completion vectors, then
504 * there will only be one completion vector available. Othewise,
505 * the number of completion vector available will be the number of
506 * available CPUs divide it by the number of devices in the
507 * local NUMA node.
508 */
509 if (cpumask_weight(&entry->comp_vect_mask) == 1) {
510 possible_cpus_comp_vect = 1;
511 dd_dev_warn(dd,
512 "Number of kernel receive queues is too large for completion vector affinity to be effective\n");
513 } else {
514 possible_cpus_comp_vect +=
515 cpumask_weight(&entry->comp_vect_mask) /
516 hfi1_per_node_cntr[dd->node];
517
518 /*
519 * If the completion vector CPUs available doesn't divide
520 * evenly among devices, then the first device device to be
521 * initialized gets an extra CPU.
522 */
523 if (first_dev_init &&
524 cpumask_weight(&entry->comp_vect_mask) %
525 hfi1_per_node_cntr[dd->node] != 0)
526 possible_cpus_comp_vect++;
527 }
528
529 dd->comp_vect_possible_cpus = possible_cpus_comp_vect;
530
531 /* Reserving CPUs for device completion vector */
532 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
533 curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask,
534 entry->comp_vect_affinity);
535 if (curr_cpu < 0)
536 goto fail;
537
538 cpumask_set_cpu(curr_cpu, dev_comp_vect_mask);
539 }
540
541 hfi1_cdbg(AFFINITY,
542 "[%s] Completion vector affinity CPU set(s) %*pbl",
543 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi),
544 cpumask_pr_args(dev_comp_vect_mask));
545
546 return 0;
547
548fail:
549 for (j = 0; j < i; j++)
550 per_cpu_affinity_put_max(&entry->comp_vect_mask,
551 entry->comp_vect_affinity);
552
553 return curr_cpu;
554}
555
556/*
557 * It assumes dd->comp_vect_possible_cpus is available.
558 */
559static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd,
560 struct hfi1_affinity_node *entry)
561 __must_hold(&node_affinity.lock)
562{
563 int i, cpu;
564
565 lockdep_assert_held(&node_affinity.lock);
566 if (!dd->comp_vect_possible_cpus)
567 return;
568
569 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
570 cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask,
571 entry->comp_vect_affinity);
572 /* Clearing CPU in device completion vector cpu mask */
573 if (cpu >= 0)
574 cpumask_clear_cpu(cpu, &dd->comp_vect->mask);
575 }
576
577 dd->comp_vect_possible_cpus = 0;
578}
579
580/*
581 * Interrupt affinity.
582 *
583 * non-rcv avail gets a default mask that
584 * starts as possible cpus with threads reset
585 * and each rcv avail reset.
586 *
587 * rcv avail gets node relative 1 wrapping back
588 * to the node relative 1 as necessary.
589 *
590 */
591int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
592{
593 struct hfi1_affinity_node *entry;
594 const struct cpumask *local_mask;
595 int curr_cpu, possible, i, ret;
596 bool new_entry = false;
597
598 local_mask = cpumask_of_node(dd->node);
599 if (cpumask_first(local_mask) >= nr_cpu_ids)
600 local_mask = topology_core_cpumask(0);
601
602 mutex_lock(&node_affinity.lock);
603 entry = node_affinity_lookup(dd->node);
604
605 /*
606 * If this is the first time this NUMA node's affinity is used,
607 * create an entry in the global affinity structure and initialize it.
608 */
609 if (!entry) {
610 entry = node_affinity_allocate(dd->node);
611 if (!entry) {
612 dd_dev_err(dd,
613 "Unable to allocate global affinity node\n");
614 ret = -ENOMEM;
615 goto fail;
616 }
617 new_entry = true;
618
619 init_cpu_mask_set(&entry->def_intr);
620 init_cpu_mask_set(&entry->rcv_intr);
621 cpumask_clear(&entry->comp_vect_mask);
622 cpumask_clear(&entry->general_intr_mask);
623 /* Use the "real" cpu mask of this node as the default */
624 cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
625 local_mask);
626
627 /* fill in the receive list */
628 possible = cpumask_weight(&entry->def_intr.mask);
629 curr_cpu = cpumask_first(&entry->def_intr.mask);
630
631 if (possible == 1) {
632 /* only one CPU, everyone will use it */
633 cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
634 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
635 } else {
636 /*
637 * The general/control context will be the first CPU in
638 * the default list, so it is removed from the default
639 * list and added to the general interrupt list.
640 */
641 cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
642 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
643 curr_cpu = cpumask_next(curr_cpu,
644 &entry->def_intr.mask);
645
646 /*
647 * Remove the remaining kernel receive queues from
648 * the default list and add them to the receive list.
649 */
650 for (i = 0;
651 i < (dd->n_krcv_queues - 1) *
652 hfi1_per_node_cntr[dd->node];
653 i++) {
654 cpumask_clear_cpu(curr_cpu,
655 &entry->def_intr.mask);
656 cpumask_set_cpu(curr_cpu,
657 &entry->rcv_intr.mask);
658 curr_cpu = cpumask_next(curr_cpu,
659 &entry->def_intr.mask);
660 if (curr_cpu >= nr_cpu_ids)
661 break;
662 }
663
664 /*
665 * If there ends up being 0 CPU cores leftover for SDMA
666 * engines, use the same CPU cores as general/control
667 * context.
668 */
669 if (cpumask_empty(&entry->def_intr.mask))
670 cpumask_copy(&entry->def_intr.mask,
671 &entry->general_intr_mask);
672 }
673
674 /* Determine completion vector CPUs for the entire node */
675 cpumask_and(&entry->comp_vect_mask,
676 &node_affinity.real_cpu_mask, local_mask);
677 cpumask_andnot(&entry->comp_vect_mask,
678 &entry->comp_vect_mask,
679 &entry->rcv_intr.mask);
680 cpumask_andnot(&entry->comp_vect_mask,
681 &entry->comp_vect_mask,
682 &entry->general_intr_mask);
683
684 /*
685 * If there ends up being 0 CPU cores leftover for completion
686 * vectors, use the same CPU core as the general/control
687 * context.
688 */
689 if (cpumask_empty(&entry->comp_vect_mask))
690 cpumask_copy(&entry->comp_vect_mask,
691 &entry->general_intr_mask);
692 }
693
694 ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry);
695 if (ret < 0)
696 goto fail;
697
698 if (new_entry)
699 node_affinity_add_tail(entry);
700
701 dd->affinity_entry = entry;
702 mutex_unlock(&node_affinity.lock);
703
704 return 0;
705
706fail:
707 if (new_entry)
708 node_affinity_destroy(entry);
709 mutex_unlock(&node_affinity.lock);
710 return ret;
711}
712
713void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)
714{
715 struct hfi1_affinity_node *entry;
716
717 mutex_lock(&node_affinity.lock);
718 if (!dd->affinity_entry)
719 goto unlock;
720 entry = node_affinity_lookup(dd->node);
721 if (!entry)
722 goto unlock;
723
724 /*
725 * Free device completion vector CPUs to be used by future
726 * completion vectors
727 */
728 _dev_comp_vect_cpu_mask_clean_up(dd, entry);
729unlock:
730 dd->affinity_entry = NULL;
731 mutex_unlock(&node_affinity.lock);
732}
733
734/*
735 * Function updates the irq affinity hint for msix after it has been changed
736 * by the user using the /proc/irq interface. This function only accepts
737 * one cpu in the mask.
738 */
739static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)
740{
741 struct sdma_engine *sde = msix->arg;
742 struct hfi1_devdata *dd = sde->dd;
743 struct hfi1_affinity_node *entry;
744 struct cpu_mask_set *set;
745 int i, old_cpu;
746
747 if (cpu > num_online_cpus() || cpu == sde->cpu)
748 return;
749
750 mutex_lock(&node_affinity.lock);
751 entry = node_affinity_lookup(dd->node);
752 if (!entry)
753 goto unlock;
754
755 old_cpu = sde->cpu;
756 sde->cpu = cpu;
757 cpumask_clear(&msix->mask);
758 cpumask_set_cpu(cpu, &msix->mask);
759 dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n",
760 msix->irq, irq_type_names[msix->type],
761 sde->this_idx, cpu);
762 irq_set_affinity_hint(msix->irq, &msix->mask);
763
764 /*
765 * Set the new cpu in the hfi1_affinity_node and clean
766 * the old cpu if it is not used by any other IRQ
767 */
768 set = &entry->def_intr;
769 cpumask_set_cpu(cpu, &set->mask);
770 cpumask_set_cpu(cpu, &set->used);
771 for (i = 0; i < dd->msix_info.max_requested; i++) {
772 struct hfi1_msix_entry *other_msix;
773
774 other_msix = &dd->msix_info.msix_entries[i];
775 if (other_msix->type != IRQ_SDMA || other_msix == msix)
776 continue;
777
778 if (cpumask_test_cpu(old_cpu, &other_msix->mask))
779 goto unlock;
780 }
781 cpumask_clear_cpu(old_cpu, &set->mask);
782 cpumask_clear_cpu(old_cpu, &set->used);
783unlock:
784 mutex_unlock(&node_affinity.lock);
785}
786
787static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify,
788 const cpumask_t *mask)
789{
790 int cpu = cpumask_first(mask);
791 struct hfi1_msix_entry *msix = container_of(notify,
792 struct hfi1_msix_entry,
793 notify);
794
795 /* Only one CPU configuration supported currently */
796 hfi1_update_sdma_affinity(msix, cpu);
797}
798
799static void hfi1_irq_notifier_release(struct kref *ref)
800{
801 /*
802 * This is required by affinity notifier. We don't have anything to
803 * free here.
804 */
805}
806
807static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)
808{
809 struct irq_affinity_notify *notify = &msix->notify;
810
811 notify->irq = msix->irq;
812 notify->notify = hfi1_irq_notifier_notify;
813 notify->release = hfi1_irq_notifier_release;
814
815 if (irq_set_affinity_notifier(notify->irq, notify))
816 pr_err("Failed to register sdma irq affinity notifier for irq %d\n",
817 notify->irq);
818}
819
820static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix)
821{
822 struct irq_affinity_notify *notify = &msix->notify;
823
824 if (irq_set_affinity_notifier(notify->irq, NULL))
825 pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n",
826 notify->irq);
827}
828
829/*
830 * Function sets the irq affinity for msix.
831 * It *must* be called with node_affinity.lock held.
832 */
833static int get_irq_affinity(struct hfi1_devdata *dd,
834 struct hfi1_msix_entry *msix)
835{
836 cpumask_var_t diff;
837 struct hfi1_affinity_node *entry;
838 struct cpu_mask_set *set = NULL;
839 struct sdma_engine *sde = NULL;
840 struct hfi1_ctxtdata *rcd = NULL;
841 char extra[64];
842 int cpu = -1;
843
844 extra[0] = '\0';
845 cpumask_clear(&msix->mask);
846
847 entry = node_affinity_lookup(dd->node);
848
849 switch (msix->type) {
850 case IRQ_SDMA:
851 sde = (struct sdma_engine *)msix->arg;
852 scnprintf(extra, 64, "engine %u", sde->this_idx);
853 set = &entry->def_intr;
854 break;
855 case IRQ_GENERAL:
856 cpu = cpumask_first(&entry->general_intr_mask);
857 break;
858 case IRQ_RCVCTXT:
859 rcd = (struct hfi1_ctxtdata *)msix->arg;
860 if (rcd->ctxt == HFI1_CTRL_CTXT)
861 cpu = cpumask_first(&entry->general_intr_mask);
862 else
863 set = &entry->rcv_intr;
864 scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
865 break;
866 case IRQ_NETDEVCTXT:
867 rcd = (struct hfi1_ctxtdata *)msix->arg;
868 set = &entry->def_intr;
869 scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
870 break;
871 default:
872 dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);
873 return -EINVAL;
874 }
875
876 /*
877 * The general and control contexts are placed on a particular
878 * CPU, which is set above. Skip accounting for it. Everything else
879 * finds its CPU here.
880 */
881 if (cpu == -1 && set) {
882 if (!zalloc_cpumask_var(&diff, GFP_KERNEL))
883 return -ENOMEM;
884
885 cpu = cpu_mask_set_get_first(set, diff);
886 if (cpu < 0) {
887 free_cpumask_var(diff);
888 dd_dev_err(dd, "Failure to obtain CPU for IRQ\n");
889 return cpu;
890 }
891
892 free_cpumask_var(diff);
893 }
894
895 cpumask_set_cpu(cpu, &msix->mask);
896 dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n",
897 msix->irq, irq_type_names[msix->type],
898 extra, cpu);
899 irq_set_affinity_hint(msix->irq, &msix->mask);
900
901 if (msix->type == IRQ_SDMA) {
902 sde->cpu = cpu;
903 hfi1_setup_sdma_notifier(msix);
904 }
905
906 return 0;
907}
908
909int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
910{
911 int ret;
912
913 mutex_lock(&node_affinity.lock);
914 ret = get_irq_affinity(dd, msix);
915 mutex_unlock(&node_affinity.lock);
916 return ret;
917}
918
919void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
920 struct hfi1_msix_entry *msix)
921{
922 struct cpu_mask_set *set = NULL;
923 struct hfi1_affinity_node *entry;
924
925 mutex_lock(&node_affinity.lock);
926 entry = node_affinity_lookup(dd->node);
927
928 switch (msix->type) {
929 case IRQ_SDMA:
930 set = &entry->def_intr;
931 hfi1_cleanup_sdma_notifier(msix);
932 break;
933 case IRQ_GENERAL:
934 /* Don't do accounting for general contexts */
935 break;
936 case IRQ_RCVCTXT: {
937 struct hfi1_ctxtdata *rcd = msix->arg;
938
939 /* Don't do accounting for control contexts */
940 if (rcd->ctxt != HFI1_CTRL_CTXT)
941 set = &entry->rcv_intr;
942 break;
943 }
944 case IRQ_NETDEVCTXT:
945 set = &entry->def_intr;
946 break;
947 default:
948 mutex_unlock(&node_affinity.lock);
949 return;
950 }
951
952 if (set) {
953 cpumask_andnot(&set->used, &set->used, &msix->mask);
954 _cpu_mask_set_gen_dec(set);
955 }
956
957 irq_set_affinity_hint(msix->irq, NULL);
958 cpumask_clear(&msix->mask);
959 mutex_unlock(&node_affinity.lock);
960}
961
962/* This should be called with node_affinity.lock held */
963static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
964 struct hfi1_affinity_node_list *affinity)
965{
966 int possible, curr_cpu, i;
967 uint num_cores_per_socket = node_affinity.num_online_cpus /
968 affinity->num_core_siblings /
969 node_affinity.num_online_nodes;
970
971 cpumask_copy(hw_thread_mask, &affinity->proc.mask);
972 if (affinity->num_core_siblings > 0) {
973 /* Removing other siblings not needed for now */
974 possible = cpumask_weight(hw_thread_mask);
975 curr_cpu = cpumask_first(hw_thread_mask);
976 for (i = 0;
977 i < num_cores_per_socket * node_affinity.num_online_nodes;
978 i++)
979 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
980
981 for (; i < possible; i++) {
982 cpumask_clear_cpu(curr_cpu, hw_thread_mask);
983 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
984 }
985
986 /* Identifying correct HW threads within physical cores */
987 cpumask_shift_left(hw_thread_mask, hw_thread_mask,
988 num_cores_per_socket *
989 node_affinity.num_online_nodes *
990 hw_thread_no);
991 }
992}
993
994int hfi1_get_proc_affinity(int node)
995{
996 int cpu = -1, ret, i;
997 struct hfi1_affinity_node *entry;
998 cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
999 const struct cpumask *node_mask,
1000 *proc_mask = current->cpus_ptr;
1001 struct hfi1_affinity_node_list *affinity = &node_affinity;
1002 struct cpu_mask_set *set = &affinity->proc;
1003
1004 /*
1005 * check whether process/context affinity has already
1006 * been set
1007 */
1008 if (current->nr_cpus_allowed == 1) {
1009 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",
1010 current->pid, current->comm,
1011 cpumask_pr_args(proc_mask));
1012 /*
1013 * Mark the pre-set CPU as used. This is atomic so we don't
1014 * need the lock
1015 */
1016 cpu = cpumask_first(proc_mask);
1017 cpumask_set_cpu(cpu, &set->used);
1018 goto done;
1019 } else if (current->nr_cpus_allowed < cpumask_weight(&set->mask)) {
1020 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",
1021 current->pid, current->comm,
1022 cpumask_pr_args(proc_mask));
1023 goto done;
1024 }
1025
1026 /*
1027 * The process does not have a preset CPU affinity so find one to
1028 * recommend using the following algorithm:
1029 *
1030 * For each user process that is opening a context on HFI Y:
1031 * a) If all cores are filled, reinitialize the bitmask
1032 * b) Fill real cores first, then HT cores (First set of HT
1033 * cores on all physical cores, then second set of HT core,
1034 * and, so on) in the following order:
1035 *
1036 * 1. Same NUMA node as HFI Y and not running an IRQ
1037 * handler
1038 * 2. Same NUMA node as HFI Y and running an IRQ handler
1039 * 3. Different NUMA node to HFI Y and not running an IRQ
1040 * handler
1041 * 4. Different NUMA node to HFI Y and running an IRQ
1042 * handler
1043 * c) Mark core as filled in the bitmask. As user processes are
1044 * done, clear cores from the bitmask.
1045 */
1046
1047 ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
1048 if (!ret)
1049 goto done;
1050 ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
1051 if (!ret)
1052 goto free_diff;
1053 ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
1054 if (!ret)
1055 goto free_hw_thread_mask;
1056 ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
1057 if (!ret)
1058 goto free_available_mask;
1059
1060 mutex_lock(&affinity->lock);
1061 /*
1062 * If we've used all available HW threads, clear the mask and start
1063 * overloading.
1064 */
1065 _cpu_mask_set_gen_inc(set);
1066
1067 /*
1068 * If NUMA node has CPUs used by interrupt handlers, include them in the
1069 * interrupt handler mask.
1070 */
1071 entry = node_affinity_lookup(node);
1072 if (entry) {
1073 cpumask_copy(intrs_mask, (entry->def_intr.gen ?
1074 &entry->def_intr.mask :
1075 &entry->def_intr.used));
1076 cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
1077 &entry->rcv_intr.mask :
1078 &entry->rcv_intr.used));
1079 cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
1080 }
1081 hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
1082 cpumask_pr_args(intrs_mask));
1083
1084 cpumask_copy(hw_thread_mask, &set->mask);
1085
1086 /*
1087 * If HT cores are enabled, identify which HW threads within the
1088 * physical cores should be used.
1089 */
1090 if (affinity->num_core_siblings > 0) {
1091 for (i = 0; i < affinity->num_core_siblings; i++) {
1092 find_hw_thread_mask(i, hw_thread_mask, affinity);
1093
1094 /*
1095 * If there's at least one available core for this HW
1096 * thread number, stop looking for a core.
1097 *
1098 * diff will always be not empty at least once in this
1099 * loop as the used mask gets reset when
1100 * (set->mask == set->used) before this loop.
1101 */
1102 cpumask_andnot(diff, hw_thread_mask, &set->used);
1103 if (!cpumask_empty(diff))
1104 break;
1105 }
1106 }
1107 hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
1108 cpumask_pr_args(hw_thread_mask));
1109
1110 node_mask = cpumask_of_node(node);
1111 hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
1112 cpumask_pr_args(node_mask));
1113
1114 /* Get cpumask of available CPUs on preferred NUMA */
1115 cpumask_and(available_mask, hw_thread_mask, node_mask);
1116 cpumask_andnot(available_mask, available_mask, &set->used);
1117 hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
1118 cpumask_pr_args(available_mask));
1119
1120 /*
1121 * At first, we don't want to place processes on the same
1122 * CPUs as interrupt handlers. Then, CPUs running interrupt
1123 * handlers are used.
1124 *
1125 * 1) If diff is not empty, then there are CPUs not running
1126 * non-interrupt handlers available, so diff gets copied
1127 * over to available_mask.
1128 * 2) If diff is empty, then all CPUs not running interrupt
1129 * handlers are taken, so available_mask contains all
1130 * available CPUs running interrupt handlers.
1131 * 3) If available_mask is empty, then all CPUs on the
1132 * preferred NUMA node are taken, so other NUMA nodes are
1133 * used for process assignments using the same method as
1134 * the preferred NUMA node.
1135 */
1136 cpumask_andnot(diff, available_mask, intrs_mask);
1137 if (!cpumask_empty(diff))
1138 cpumask_copy(available_mask, diff);
1139
1140 /* If we don't have CPUs on the preferred node, use other NUMA nodes */
1141 if (cpumask_empty(available_mask)) {
1142 cpumask_andnot(available_mask, hw_thread_mask, &set->used);
1143 /* Excluding preferred NUMA cores */
1144 cpumask_andnot(available_mask, available_mask, node_mask);
1145 hfi1_cdbg(PROC,
1146 "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
1147 cpumask_pr_args(available_mask));
1148
1149 /*
1150 * At first, we don't want to place processes on the same
1151 * CPUs as interrupt handlers.
1152 */
1153 cpumask_andnot(diff, available_mask, intrs_mask);
1154 if (!cpumask_empty(diff))
1155 cpumask_copy(available_mask, diff);
1156 }
1157 hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
1158 cpumask_pr_args(available_mask));
1159
1160 cpu = cpumask_first(available_mask);
1161 if (cpu >= nr_cpu_ids) /* empty */
1162 cpu = -1;
1163 else
1164 cpumask_set_cpu(cpu, &set->used);
1165
1166 mutex_unlock(&affinity->lock);
1167 hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
1168
1169 free_cpumask_var(intrs_mask);
1170free_available_mask:
1171 free_cpumask_var(available_mask);
1172free_hw_thread_mask:
1173 free_cpumask_var(hw_thread_mask);
1174free_diff:
1175 free_cpumask_var(diff);
1176done:
1177 return cpu;
1178}
1179
1180void hfi1_put_proc_affinity(int cpu)
1181{
1182 struct hfi1_affinity_node_list *affinity = &node_affinity;
1183 struct cpu_mask_set *set = &affinity->proc;
1184
1185 if (cpu < 0)
1186 return;
1187
1188 mutex_lock(&affinity->lock);
1189 cpu_mask_set_put(set, cpu);
1190 hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
1191 mutex_unlock(&affinity->lock);
1192}
1/*
2 * Copyright(c) 2015 - 2020 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47#include <linux/topology.h>
48#include <linux/cpumask.h>
49#include <linux/module.h>
50#include <linux/interrupt.h>
51#include <linux/numa.h>
52
53#include "hfi.h"
54#include "affinity.h"
55#include "sdma.h"
56#include "trace.h"
57
58struct hfi1_affinity_node_list node_affinity = {
59 .list = LIST_HEAD_INIT(node_affinity.list),
60 .lock = __MUTEX_INITIALIZER(node_affinity.lock)
61};
62
63/* Name of IRQ types, indexed by enum irq_type */
64static const char * const irq_type_names[] = {
65 "SDMA",
66 "RCVCTXT",
67 "NETDEVCTXT",
68 "GENERAL",
69 "OTHER",
70};
71
72/* Per NUMA node count of HFI devices */
73static unsigned int *hfi1_per_node_cntr;
74
75static inline void init_cpu_mask_set(struct cpu_mask_set *set)
76{
77 cpumask_clear(&set->mask);
78 cpumask_clear(&set->used);
79 set->gen = 0;
80}
81
82/* Increment generation of CPU set if needed */
83static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set)
84{
85 if (cpumask_equal(&set->mask, &set->used)) {
86 /*
87 * We've used up all the CPUs, bump up the generation
88 * and reset the 'used' map
89 */
90 set->gen++;
91 cpumask_clear(&set->used);
92 }
93}
94
95static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set)
96{
97 if (cpumask_empty(&set->used) && set->gen) {
98 set->gen--;
99 cpumask_copy(&set->used, &set->mask);
100 }
101}
102
103/* Get the first CPU from the list of unused CPUs in a CPU set data structure */
104static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff)
105{
106 int cpu;
107
108 if (!diff || !set)
109 return -EINVAL;
110
111 _cpu_mask_set_gen_inc(set);
112
113 /* Find out CPUs left in CPU mask */
114 cpumask_andnot(diff, &set->mask, &set->used);
115
116 cpu = cpumask_first(diff);
117 if (cpu >= nr_cpu_ids) /* empty */
118 cpu = -EINVAL;
119 else
120 cpumask_set_cpu(cpu, &set->used);
121
122 return cpu;
123}
124
125static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)
126{
127 if (!set)
128 return;
129
130 cpumask_clear_cpu(cpu, &set->used);
131 _cpu_mask_set_gen_dec(set);
132}
133
134/* Initialize non-HT cpu cores mask */
135void init_real_cpu_mask(void)
136{
137 int possible, curr_cpu, i, ht;
138
139 cpumask_clear(&node_affinity.real_cpu_mask);
140
141 /* Start with cpu online mask as the real cpu mask */
142 cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
143
144 /*
145 * Remove HT cores from the real cpu mask. Do this in two steps below.
146 */
147 possible = cpumask_weight(&node_affinity.real_cpu_mask);
148 ht = cpumask_weight(topology_sibling_cpumask(
149 cpumask_first(&node_affinity.real_cpu_mask)));
150 /*
151 * Step 1. Skip over the first N HT siblings and use them as the
152 * "real" cores. Assumes that HT cores are not enumerated in
153 * succession (except in the single core case).
154 */
155 curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
156 for (i = 0; i < possible / ht; i++)
157 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
158 /*
159 * Step 2. Remove the remaining HT siblings. Use cpumask_next() to
160 * skip any gaps.
161 */
162 for (; i < possible; i++) {
163 cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
164 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
165 }
166}
167
168int node_affinity_init(void)
169{
170 int node;
171 struct pci_dev *dev = NULL;
172 const struct pci_device_id *ids = hfi1_pci_tbl;
173
174 cpumask_clear(&node_affinity.proc.used);
175 cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
176
177 node_affinity.proc.gen = 0;
178 node_affinity.num_core_siblings =
179 cpumask_weight(topology_sibling_cpumask(
180 cpumask_first(&node_affinity.proc.mask)
181 ));
182 node_affinity.num_possible_nodes = num_possible_nodes();
183 node_affinity.num_online_nodes = num_online_nodes();
184 node_affinity.num_online_cpus = num_online_cpus();
185
186 /*
187 * The real cpu mask is part of the affinity struct but it has to be
188 * initialized early. It is needed to calculate the number of user
189 * contexts in set_up_context_variables().
190 */
191 init_real_cpu_mask();
192
193 hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes,
194 sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
195 if (!hfi1_per_node_cntr)
196 return -ENOMEM;
197
198 while (ids->vendor) {
199 dev = NULL;
200 while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
201 node = pcibus_to_node(dev->bus);
202 if (node < 0)
203 goto out;
204
205 hfi1_per_node_cntr[node]++;
206 }
207 ids++;
208 }
209
210 return 0;
211
212out:
213 /*
214 * Invalid PCI NUMA node information found, note it, and populate
215 * our database 1:1.
216 */
217 pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n");
218 pr_err("HFI: System BIOS may need to be upgraded\n");
219 for (node = 0; node < node_affinity.num_possible_nodes; node++)
220 hfi1_per_node_cntr[node] = 1;
221
222 return 0;
223}
224
225static void node_affinity_destroy(struct hfi1_affinity_node *entry)
226{
227 free_percpu(entry->comp_vect_affinity);
228 kfree(entry);
229}
230
231void node_affinity_destroy_all(void)
232{
233 struct list_head *pos, *q;
234 struct hfi1_affinity_node *entry;
235
236 mutex_lock(&node_affinity.lock);
237 list_for_each_safe(pos, q, &node_affinity.list) {
238 entry = list_entry(pos, struct hfi1_affinity_node,
239 list);
240 list_del(pos);
241 node_affinity_destroy(entry);
242 }
243 mutex_unlock(&node_affinity.lock);
244 kfree(hfi1_per_node_cntr);
245}
246
247static struct hfi1_affinity_node *node_affinity_allocate(int node)
248{
249 struct hfi1_affinity_node *entry;
250
251 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
252 if (!entry)
253 return NULL;
254 entry->node = node;
255 entry->comp_vect_affinity = alloc_percpu(u16);
256 INIT_LIST_HEAD(&entry->list);
257
258 return entry;
259}
260
261/*
262 * It appends an entry to the list.
263 * It *must* be called with node_affinity.lock held.
264 */
265static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
266{
267 list_add_tail(&entry->list, &node_affinity.list);
268}
269
270/* It must be called with node_affinity.lock held */
271static struct hfi1_affinity_node *node_affinity_lookup(int node)
272{
273 struct list_head *pos;
274 struct hfi1_affinity_node *entry;
275
276 list_for_each(pos, &node_affinity.list) {
277 entry = list_entry(pos, struct hfi1_affinity_node, list);
278 if (entry->node == node)
279 return entry;
280 }
281
282 return NULL;
283}
284
285static int per_cpu_affinity_get(cpumask_var_t possible_cpumask,
286 u16 __percpu *comp_vect_affinity)
287{
288 int curr_cpu;
289 u16 cntr;
290 u16 prev_cntr;
291 int ret_cpu;
292
293 if (!possible_cpumask) {
294 ret_cpu = -EINVAL;
295 goto fail;
296 }
297
298 if (!comp_vect_affinity) {
299 ret_cpu = -EINVAL;
300 goto fail;
301 }
302
303 ret_cpu = cpumask_first(possible_cpumask);
304 if (ret_cpu >= nr_cpu_ids) {
305 ret_cpu = -EINVAL;
306 goto fail;
307 }
308
309 prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu);
310 for_each_cpu(curr_cpu, possible_cpumask) {
311 cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
312
313 if (cntr < prev_cntr) {
314 ret_cpu = curr_cpu;
315 prev_cntr = cntr;
316 }
317 }
318
319 *per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1;
320
321fail:
322 return ret_cpu;
323}
324
325static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask,
326 u16 __percpu *comp_vect_affinity)
327{
328 int curr_cpu;
329 int max_cpu;
330 u16 cntr;
331 u16 prev_cntr;
332
333 if (!possible_cpumask)
334 return -EINVAL;
335
336 if (!comp_vect_affinity)
337 return -EINVAL;
338
339 max_cpu = cpumask_first(possible_cpumask);
340 if (max_cpu >= nr_cpu_ids)
341 return -EINVAL;
342
343 prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu);
344 for_each_cpu(curr_cpu, possible_cpumask) {
345 cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
346
347 if (cntr > prev_cntr) {
348 max_cpu = curr_cpu;
349 prev_cntr = cntr;
350 }
351 }
352
353 *per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1;
354
355 return max_cpu;
356}
357
358/*
359 * Non-interrupt CPUs are used first, then interrupt CPUs.
360 * Two already allocated cpu masks must be passed.
361 */
362static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,
363 struct hfi1_affinity_node *entry,
364 cpumask_var_t non_intr_cpus,
365 cpumask_var_t available_cpus)
366 __must_hold(&node_affinity.lock)
367{
368 int cpu;
369 struct cpu_mask_set *set = dd->comp_vect;
370
371 lockdep_assert_held(&node_affinity.lock);
372 if (!non_intr_cpus) {
373 cpu = -1;
374 goto fail;
375 }
376
377 if (!available_cpus) {
378 cpu = -1;
379 goto fail;
380 }
381
382 /* Available CPUs for pinning completion vectors */
383 _cpu_mask_set_gen_inc(set);
384 cpumask_andnot(available_cpus, &set->mask, &set->used);
385
386 /* Available CPUs without SDMA engine interrupts */
387 cpumask_andnot(non_intr_cpus, available_cpus,
388 &entry->def_intr.used);
389
390 /* If there are non-interrupt CPUs available, use them first */
391 if (!cpumask_empty(non_intr_cpus))
392 cpu = cpumask_first(non_intr_cpus);
393 else /* Otherwise, use interrupt CPUs */
394 cpu = cpumask_first(available_cpus);
395
396 if (cpu >= nr_cpu_ids) { /* empty */
397 cpu = -1;
398 goto fail;
399 }
400 cpumask_set_cpu(cpu, &set->used);
401
402fail:
403 return cpu;
404}
405
406static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu)
407{
408 struct cpu_mask_set *set = dd->comp_vect;
409
410 if (cpu < 0)
411 return;
412
413 cpu_mask_set_put(set, cpu);
414}
415
416/* _dev_comp_vect_mappings_destroy() is reentrant */
417static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
418{
419 int i, cpu;
420
421 if (!dd->comp_vect_mappings)
422 return;
423
424 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
425 cpu = dd->comp_vect_mappings[i];
426 _dev_comp_vect_cpu_put(dd, cpu);
427 dd->comp_vect_mappings[i] = -1;
428 hfi1_cdbg(AFFINITY,
429 "[%s] Release CPU %d from completion vector %d",
430 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i);
431 }
432
433 kfree(dd->comp_vect_mappings);
434 dd->comp_vect_mappings = NULL;
435}
436
437/*
438 * This function creates the table for looking up CPUs for completion vectors.
439 * num_comp_vectors needs to have been initilized before calling this function.
440 */
441static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
442 struct hfi1_affinity_node *entry)
443 __must_hold(&node_affinity.lock)
444{
445 int i, cpu, ret;
446 cpumask_var_t non_intr_cpus;
447 cpumask_var_t available_cpus;
448
449 lockdep_assert_held(&node_affinity.lock);
450
451 if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL))
452 return -ENOMEM;
453
454 if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) {
455 free_cpumask_var(non_intr_cpus);
456 return -ENOMEM;
457 }
458
459 dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus,
460 sizeof(*dd->comp_vect_mappings),
461 GFP_KERNEL);
462 if (!dd->comp_vect_mappings) {
463 ret = -ENOMEM;
464 goto fail;
465 }
466 for (i = 0; i < dd->comp_vect_possible_cpus; i++)
467 dd->comp_vect_mappings[i] = -1;
468
469 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
470 cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus,
471 available_cpus);
472 if (cpu < 0) {
473 ret = -EINVAL;
474 goto fail;
475 }
476
477 dd->comp_vect_mappings[i] = cpu;
478 hfi1_cdbg(AFFINITY,
479 "[%s] Completion Vector %d -> CPU %d",
480 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu);
481 }
482
483 free_cpumask_var(available_cpus);
484 free_cpumask_var(non_intr_cpus);
485 return 0;
486
487fail:
488 free_cpumask_var(available_cpus);
489 free_cpumask_var(non_intr_cpus);
490 _dev_comp_vect_mappings_destroy(dd);
491
492 return ret;
493}
494
495int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd)
496{
497 int ret;
498 struct hfi1_affinity_node *entry;
499
500 mutex_lock(&node_affinity.lock);
501 entry = node_affinity_lookup(dd->node);
502 if (!entry) {
503 ret = -EINVAL;
504 goto unlock;
505 }
506 ret = _dev_comp_vect_mappings_create(dd, entry);
507unlock:
508 mutex_unlock(&node_affinity.lock);
509
510 return ret;
511}
512
513void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd)
514{
515 _dev_comp_vect_mappings_destroy(dd);
516}
517
518int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect)
519{
520 struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
521 struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
522
523 if (!dd->comp_vect_mappings)
524 return -EINVAL;
525 if (comp_vect >= dd->comp_vect_possible_cpus)
526 return -EINVAL;
527
528 return dd->comp_vect_mappings[comp_vect];
529}
530
531/*
532 * It assumes dd->comp_vect_possible_cpus is available.
533 */
534static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd,
535 struct hfi1_affinity_node *entry,
536 bool first_dev_init)
537 __must_hold(&node_affinity.lock)
538{
539 int i, j, curr_cpu;
540 int possible_cpus_comp_vect = 0;
541 struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask;
542
543 lockdep_assert_held(&node_affinity.lock);
544 /*
545 * If there's only one CPU available for completion vectors, then
546 * there will only be one completion vector available. Othewise,
547 * the number of completion vector available will be the number of
548 * available CPUs divide it by the number of devices in the
549 * local NUMA node.
550 */
551 if (cpumask_weight(&entry->comp_vect_mask) == 1) {
552 possible_cpus_comp_vect = 1;
553 dd_dev_warn(dd,
554 "Number of kernel receive queues is too large for completion vector affinity to be effective\n");
555 } else {
556 possible_cpus_comp_vect +=
557 cpumask_weight(&entry->comp_vect_mask) /
558 hfi1_per_node_cntr[dd->node];
559
560 /*
561 * If the completion vector CPUs available doesn't divide
562 * evenly among devices, then the first device device to be
563 * initialized gets an extra CPU.
564 */
565 if (first_dev_init &&
566 cpumask_weight(&entry->comp_vect_mask) %
567 hfi1_per_node_cntr[dd->node] != 0)
568 possible_cpus_comp_vect++;
569 }
570
571 dd->comp_vect_possible_cpus = possible_cpus_comp_vect;
572
573 /* Reserving CPUs for device completion vector */
574 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
575 curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask,
576 entry->comp_vect_affinity);
577 if (curr_cpu < 0)
578 goto fail;
579
580 cpumask_set_cpu(curr_cpu, dev_comp_vect_mask);
581 }
582
583 hfi1_cdbg(AFFINITY,
584 "[%s] Completion vector affinity CPU set(s) %*pbl",
585 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi),
586 cpumask_pr_args(dev_comp_vect_mask));
587
588 return 0;
589
590fail:
591 for (j = 0; j < i; j++)
592 per_cpu_affinity_put_max(&entry->comp_vect_mask,
593 entry->comp_vect_affinity);
594
595 return curr_cpu;
596}
597
598/*
599 * It assumes dd->comp_vect_possible_cpus is available.
600 */
601static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd,
602 struct hfi1_affinity_node *entry)
603 __must_hold(&node_affinity.lock)
604{
605 int i, cpu;
606
607 lockdep_assert_held(&node_affinity.lock);
608 if (!dd->comp_vect_possible_cpus)
609 return;
610
611 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
612 cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask,
613 entry->comp_vect_affinity);
614 /* Clearing CPU in device completion vector cpu mask */
615 if (cpu >= 0)
616 cpumask_clear_cpu(cpu, &dd->comp_vect->mask);
617 }
618
619 dd->comp_vect_possible_cpus = 0;
620}
621
622/*
623 * Interrupt affinity.
624 *
625 * non-rcv avail gets a default mask that
626 * starts as possible cpus with threads reset
627 * and each rcv avail reset.
628 *
629 * rcv avail gets node relative 1 wrapping back
630 * to the node relative 1 as necessary.
631 *
632 */
633int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
634{
635 int node = pcibus_to_node(dd->pcidev->bus);
636 struct hfi1_affinity_node *entry;
637 const struct cpumask *local_mask;
638 int curr_cpu, possible, i, ret;
639 bool new_entry = false;
640
641 /*
642 * If the BIOS does not have the NUMA node information set, select
643 * NUMA 0 so we get consistent performance.
644 */
645 if (node < 0) {
646 dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");
647 node = 0;
648 }
649 dd->node = node;
650
651 local_mask = cpumask_of_node(dd->node);
652 if (cpumask_first(local_mask) >= nr_cpu_ids)
653 local_mask = topology_core_cpumask(0);
654
655 mutex_lock(&node_affinity.lock);
656 entry = node_affinity_lookup(dd->node);
657
658 /*
659 * If this is the first time this NUMA node's affinity is used,
660 * create an entry in the global affinity structure and initialize it.
661 */
662 if (!entry) {
663 entry = node_affinity_allocate(node);
664 if (!entry) {
665 dd_dev_err(dd,
666 "Unable to allocate global affinity node\n");
667 ret = -ENOMEM;
668 goto fail;
669 }
670 new_entry = true;
671
672 init_cpu_mask_set(&entry->def_intr);
673 init_cpu_mask_set(&entry->rcv_intr);
674 cpumask_clear(&entry->comp_vect_mask);
675 cpumask_clear(&entry->general_intr_mask);
676 /* Use the "real" cpu mask of this node as the default */
677 cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
678 local_mask);
679
680 /* fill in the receive list */
681 possible = cpumask_weight(&entry->def_intr.mask);
682 curr_cpu = cpumask_first(&entry->def_intr.mask);
683
684 if (possible == 1) {
685 /* only one CPU, everyone will use it */
686 cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
687 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
688 } else {
689 /*
690 * The general/control context will be the first CPU in
691 * the default list, so it is removed from the default
692 * list and added to the general interrupt list.
693 */
694 cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
695 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
696 curr_cpu = cpumask_next(curr_cpu,
697 &entry->def_intr.mask);
698
699 /*
700 * Remove the remaining kernel receive queues from
701 * the default list and add them to the receive list.
702 */
703 for (i = 0;
704 i < (dd->n_krcv_queues - 1) *
705 hfi1_per_node_cntr[dd->node];
706 i++) {
707 cpumask_clear_cpu(curr_cpu,
708 &entry->def_intr.mask);
709 cpumask_set_cpu(curr_cpu,
710 &entry->rcv_intr.mask);
711 curr_cpu = cpumask_next(curr_cpu,
712 &entry->def_intr.mask);
713 if (curr_cpu >= nr_cpu_ids)
714 break;
715 }
716
717 /*
718 * If there ends up being 0 CPU cores leftover for SDMA
719 * engines, use the same CPU cores as general/control
720 * context.
721 */
722 if (cpumask_weight(&entry->def_intr.mask) == 0)
723 cpumask_copy(&entry->def_intr.mask,
724 &entry->general_intr_mask);
725 }
726
727 /* Determine completion vector CPUs for the entire node */
728 cpumask_and(&entry->comp_vect_mask,
729 &node_affinity.real_cpu_mask, local_mask);
730 cpumask_andnot(&entry->comp_vect_mask,
731 &entry->comp_vect_mask,
732 &entry->rcv_intr.mask);
733 cpumask_andnot(&entry->comp_vect_mask,
734 &entry->comp_vect_mask,
735 &entry->general_intr_mask);
736
737 /*
738 * If there ends up being 0 CPU cores leftover for completion
739 * vectors, use the same CPU core as the general/control
740 * context.
741 */
742 if (cpumask_weight(&entry->comp_vect_mask) == 0)
743 cpumask_copy(&entry->comp_vect_mask,
744 &entry->general_intr_mask);
745 }
746
747 ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry);
748 if (ret < 0)
749 goto fail;
750
751 if (new_entry)
752 node_affinity_add_tail(entry);
753
754 mutex_unlock(&node_affinity.lock);
755
756 return 0;
757
758fail:
759 if (new_entry)
760 node_affinity_destroy(entry);
761 mutex_unlock(&node_affinity.lock);
762 return ret;
763}
764
765void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)
766{
767 struct hfi1_affinity_node *entry;
768
769 if (dd->node < 0)
770 return;
771
772 mutex_lock(&node_affinity.lock);
773 entry = node_affinity_lookup(dd->node);
774 if (!entry)
775 goto unlock;
776
777 /*
778 * Free device completion vector CPUs to be used by future
779 * completion vectors
780 */
781 _dev_comp_vect_cpu_mask_clean_up(dd, entry);
782unlock:
783 mutex_unlock(&node_affinity.lock);
784 dd->node = NUMA_NO_NODE;
785}
786
787/*
788 * Function updates the irq affinity hint for msix after it has been changed
789 * by the user using the /proc/irq interface. This function only accepts
790 * one cpu in the mask.
791 */
792static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)
793{
794 struct sdma_engine *sde = msix->arg;
795 struct hfi1_devdata *dd = sde->dd;
796 struct hfi1_affinity_node *entry;
797 struct cpu_mask_set *set;
798 int i, old_cpu;
799
800 if (cpu > num_online_cpus() || cpu == sde->cpu)
801 return;
802
803 mutex_lock(&node_affinity.lock);
804 entry = node_affinity_lookup(dd->node);
805 if (!entry)
806 goto unlock;
807
808 old_cpu = sde->cpu;
809 sde->cpu = cpu;
810 cpumask_clear(&msix->mask);
811 cpumask_set_cpu(cpu, &msix->mask);
812 dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n",
813 msix->irq, irq_type_names[msix->type],
814 sde->this_idx, cpu);
815 irq_set_affinity_hint(msix->irq, &msix->mask);
816
817 /*
818 * Set the new cpu in the hfi1_affinity_node and clean
819 * the old cpu if it is not used by any other IRQ
820 */
821 set = &entry->def_intr;
822 cpumask_set_cpu(cpu, &set->mask);
823 cpumask_set_cpu(cpu, &set->used);
824 for (i = 0; i < dd->msix_info.max_requested; i++) {
825 struct hfi1_msix_entry *other_msix;
826
827 other_msix = &dd->msix_info.msix_entries[i];
828 if (other_msix->type != IRQ_SDMA || other_msix == msix)
829 continue;
830
831 if (cpumask_test_cpu(old_cpu, &other_msix->mask))
832 goto unlock;
833 }
834 cpumask_clear_cpu(old_cpu, &set->mask);
835 cpumask_clear_cpu(old_cpu, &set->used);
836unlock:
837 mutex_unlock(&node_affinity.lock);
838}
839
840static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify,
841 const cpumask_t *mask)
842{
843 int cpu = cpumask_first(mask);
844 struct hfi1_msix_entry *msix = container_of(notify,
845 struct hfi1_msix_entry,
846 notify);
847
848 /* Only one CPU configuration supported currently */
849 hfi1_update_sdma_affinity(msix, cpu);
850}
851
852static void hfi1_irq_notifier_release(struct kref *ref)
853{
854 /*
855 * This is required by affinity notifier. We don't have anything to
856 * free here.
857 */
858}
859
860static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)
861{
862 struct irq_affinity_notify *notify = &msix->notify;
863
864 notify->irq = msix->irq;
865 notify->notify = hfi1_irq_notifier_notify;
866 notify->release = hfi1_irq_notifier_release;
867
868 if (irq_set_affinity_notifier(notify->irq, notify))
869 pr_err("Failed to register sdma irq affinity notifier for irq %d\n",
870 notify->irq);
871}
872
873static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix)
874{
875 struct irq_affinity_notify *notify = &msix->notify;
876
877 if (irq_set_affinity_notifier(notify->irq, NULL))
878 pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n",
879 notify->irq);
880}
881
882/*
883 * Function sets the irq affinity for msix.
884 * It *must* be called with node_affinity.lock held.
885 */
886static int get_irq_affinity(struct hfi1_devdata *dd,
887 struct hfi1_msix_entry *msix)
888{
889 cpumask_var_t diff;
890 struct hfi1_affinity_node *entry;
891 struct cpu_mask_set *set = NULL;
892 struct sdma_engine *sde = NULL;
893 struct hfi1_ctxtdata *rcd = NULL;
894 char extra[64];
895 int cpu = -1;
896
897 extra[0] = '\0';
898 cpumask_clear(&msix->mask);
899
900 entry = node_affinity_lookup(dd->node);
901
902 switch (msix->type) {
903 case IRQ_SDMA:
904 sde = (struct sdma_engine *)msix->arg;
905 scnprintf(extra, 64, "engine %u", sde->this_idx);
906 set = &entry->def_intr;
907 break;
908 case IRQ_GENERAL:
909 cpu = cpumask_first(&entry->general_intr_mask);
910 break;
911 case IRQ_RCVCTXT:
912 rcd = (struct hfi1_ctxtdata *)msix->arg;
913 if (rcd->ctxt == HFI1_CTRL_CTXT)
914 cpu = cpumask_first(&entry->general_intr_mask);
915 else
916 set = &entry->rcv_intr;
917 scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
918 break;
919 case IRQ_NETDEVCTXT:
920 rcd = (struct hfi1_ctxtdata *)msix->arg;
921 set = &entry->def_intr;
922 scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
923 break;
924 default:
925 dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);
926 return -EINVAL;
927 }
928
929 /*
930 * The general and control contexts are placed on a particular
931 * CPU, which is set above. Skip accounting for it. Everything else
932 * finds its CPU here.
933 */
934 if (cpu == -1 && set) {
935 if (!zalloc_cpumask_var(&diff, GFP_KERNEL))
936 return -ENOMEM;
937
938 cpu = cpu_mask_set_get_first(set, diff);
939 if (cpu < 0) {
940 free_cpumask_var(diff);
941 dd_dev_err(dd, "Failure to obtain CPU for IRQ\n");
942 return cpu;
943 }
944
945 free_cpumask_var(diff);
946 }
947
948 cpumask_set_cpu(cpu, &msix->mask);
949 dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n",
950 msix->irq, irq_type_names[msix->type],
951 extra, cpu);
952 irq_set_affinity_hint(msix->irq, &msix->mask);
953
954 if (msix->type == IRQ_SDMA) {
955 sde->cpu = cpu;
956 hfi1_setup_sdma_notifier(msix);
957 }
958
959 return 0;
960}
961
962int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
963{
964 int ret;
965
966 mutex_lock(&node_affinity.lock);
967 ret = get_irq_affinity(dd, msix);
968 mutex_unlock(&node_affinity.lock);
969 return ret;
970}
971
972void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
973 struct hfi1_msix_entry *msix)
974{
975 struct cpu_mask_set *set = NULL;
976 struct hfi1_ctxtdata *rcd;
977 struct hfi1_affinity_node *entry;
978
979 mutex_lock(&node_affinity.lock);
980 entry = node_affinity_lookup(dd->node);
981
982 switch (msix->type) {
983 case IRQ_SDMA:
984 set = &entry->def_intr;
985 hfi1_cleanup_sdma_notifier(msix);
986 break;
987 case IRQ_GENERAL:
988 /* Don't do accounting for general contexts */
989 break;
990 case IRQ_RCVCTXT:
991 rcd = (struct hfi1_ctxtdata *)msix->arg;
992 /* Don't do accounting for control contexts */
993 if (rcd->ctxt != HFI1_CTRL_CTXT)
994 set = &entry->rcv_intr;
995 break;
996 case IRQ_NETDEVCTXT:
997 rcd = (struct hfi1_ctxtdata *)msix->arg;
998 set = &entry->def_intr;
999 break;
1000 default:
1001 mutex_unlock(&node_affinity.lock);
1002 return;
1003 }
1004
1005 if (set) {
1006 cpumask_andnot(&set->used, &set->used, &msix->mask);
1007 _cpu_mask_set_gen_dec(set);
1008 }
1009
1010 irq_set_affinity_hint(msix->irq, NULL);
1011 cpumask_clear(&msix->mask);
1012 mutex_unlock(&node_affinity.lock);
1013}
1014
1015/* This should be called with node_affinity.lock held */
1016static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
1017 struct hfi1_affinity_node_list *affinity)
1018{
1019 int possible, curr_cpu, i;
1020 uint num_cores_per_socket = node_affinity.num_online_cpus /
1021 affinity->num_core_siblings /
1022 node_affinity.num_online_nodes;
1023
1024 cpumask_copy(hw_thread_mask, &affinity->proc.mask);
1025 if (affinity->num_core_siblings > 0) {
1026 /* Removing other siblings not needed for now */
1027 possible = cpumask_weight(hw_thread_mask);
1028 curr_cpu = cpumask_first(hw_thread_mask);
1029 for (i = 0;
1030 i < num_cores_per_socket * node_affinity.num_online_nodes;
1031 i++)
1032 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
1033
1034 for (; i < possible; i++) {
1035 cpumask_clear_cpu(curr_cpu, hw_thread_mask);
1036 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
1037 }
1038
1039 /* Identifying correct HW threads within physical cores */
1040 cpumask_shift_left(hw_thread_mask, hw_thread_mask,
1041 num_cores_per_socket *
1042 node_affinity.num_online_nodes *
1043 hw_thread_no);
1044 }
1045}
1046
1047int hfi1_get_proc_affinity(int node)
1048{
1049 int cpu = -1, ret, i;
1050 struct hfi1_affinity_node *entry;
1051 cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
1052 const struct cpumask *node_mask,
1053 *proc_mask = current->cpus_ptr;
1054 struct hfi1_affinity_node_list *affinity = &node_affinity;
1055 struct cpu_mask_set *set = &affinity->proc;
1056
1057 /*
1058 * check whether process/context affinity has already
1059 * been set
1060 */
1061 if (current->nr_cpus_allowed == 1) {
1062 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",
1063 current->pid, current->comm,
1064 cpumask_pr_args(proc_mask));
1065 /*
1066 * Mark the pre-set CPU as used. This is atomic so we don't
1067 * need the lock
1068 */
1069 cpu = cpumask_first(proc_mask);
1070 cpumask_set_cpu(cpu, &set->used);
1071 goto done;
1072 } else if (current->nr_cpus_allowed < cpumask_weight(&set->mask)) {
1073 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",
1074 current->pid, current->comm,
1075 cpumask_pr_args(proc_mask));
1076 goto done;
1077 }
1078
1079 /*
1080 * The process does not have a preset CPU affinity so find one to
1081 * recommend using the following algorithm:
1082 *
1083 * For each user process that is opening a context on HFI Y:
1084 * a) If all cores are filled, reinitialize the bitmask
1085 * b) Fill real cores first, then HT cores (First set of HT
1086 * cores on all physical cores, then second set of HT core,
1087 * and, so on) in the following order:
1088 *
1089 * 1. Same NUMA node as HFI Y and not running an IRQ
1090 * handler
1091 * 2. Same NUMA node as HFI Y and running an IRQ handler
1092 * 3. Different NUMA node to HFI Y and not running an IRQ
1093 * handler
1094 * 4. Different NUMA node to HFI Y and running an IRQ
1095 * handler
1096 * c) Mark core as filled in the bitmask. As user processes are
1097 * done, clear cores from the bitmask.
1098 */
1099
1100 ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
1101 if (!ret)
1102 goto done;
1103 ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
1104 if (!ret)
1105 goto free_diff;
1106 ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
1107 if (!ret)
1108 goto free_hw_thread_mask;
1109 ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
1110 if (!ret)
1111 goto free_available_mask;
1112
1113 mutex_lock(&affinity->lock);
1114 /*
1115 * If we've used all available HW threads, clear the mask and start
1116 * overloading.
1117 */
1118 _cpu_mask_set_gen_inc(set);
1119
1120 /*
1121 * If NUMA node has CPUs used by interrupt handlers, include them in the
1122 * interrupt handler mask.
1123 */
1124 entry = node_affinity_lookup(node);
1125 if (entry) {
1126 cpumask_copy(intrs_mask, (entry->def_intr.gen ?
1127 &entry->def_intr.mask :
1128 &entry->def_intr.used));
1129 cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
1130 &entry->rcv_intr.mask :
1131 &entry->rcv_intr.used));
1132 cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
1133 }
1134 hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
1135 cpumask_pr_args(intrs_mask));
1136
1137 cpumask_copy(hw_thread_mask, &set->mask);
1138
1139 /*
1140 * If HT cores are enabled, identify which HW threads within the
1141 * physical cores should be used.
1142 */
1143 if (affinity->num_core_siblings > 0) {
1144 for (i = 0; i < affinity->num_core_siblings; i++) {
1145 find_hw_thread_mask(i, hw_thread_mask, affinity);
1146
1147 /*
1148 * If there's at least one available core for this HW
1149 * thread number, stop looking for a core.
1150 *
1151 * diff will always be not empty at least once in this
1152 * loop as the used mask gets reset when
1153 * (set->mask == set->used) before this loop.
1154 */
1155 cpumask_andnot(diff, hw_thread_mask, &set->used);
1156 if (!cpumask_empty(diff))
1157 break;
1158 }
1159 }
1160 hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
1161 cpumask_pr_args(hw_thread_mask));
1162
1163 node_mask = cpumask_of_node(node);
1164 hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
1165 cpumask_pr_args(node_mask));
1166
1167 /* Get cpumask of available CPUs on preferred NUMA */
1168 cpumask_and(available_mask, hw_thread_mask, node_mask);
1169 cpumask_andnot(available_mask, available_mask, &set->used);
1170 hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
1171 cpumask_pr_args(available_mask));
1172
1173 /*
1174 * At first, we don't want to place processes on the same
1175 * CPUs as interrupt handlers. Then, CPUs running interrupt
1176 * handlers are used.
1177 *
1178 * 1) If diff is not empty, then there are CPUs not running
1179 * non-interrupt handlers available, so diff gets copied
1180 * over to available_mask.
1181 * 2) If diff is empty, then all CPUs not running interrupt
1182 * handlers are taken, so available_mask contains all
1183 * available CPUs running interrupt handlers.
1184 * 3) If available_mask is empty, then all CPUs on the
1185 * preferred NUMA node are taken, so other NUMA nodes are
1186 * used for process assignments using the same method as
1187 * the preferred NUMA node.
1188 */
1189 cpumask_andnot(diff, available_mask, intrs_mask);
1190 if (!cpumask_empty(diff))
1191 cpumask_copy(available_mask, diff);
1192
1193 /* If we don't have CPUs on the preferred node, use other NUMA nodes */
1194 if (cpumask_empty(available_mask)) {
1195 cpumask_andnot(available_mask, hw_thread_mask, &set->used);
1196 /* Excluding preferred NUMA cores */
1197 cpumask_andnot(available_mask, available_mask, node_mask);
1198 hfi1_cdbg(PROC,
1199 "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
1200 cpumask_pr_args(available_mask));
1201
1202 /*
1203 * At first, we don't want to place processes on the same
1204 * CPUs as interrupt handlers.
1205 */
1206 cpumask_andnot(diff, available_mask, intrs_mask);
1207 if (!cpumask_empty(diff))
1208 cpumask_copy(available_mask, diff);
1209 }
1210 hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
1211 cpumask_pr_args(available_mask));
1212
1213 cpu = cpumask_first(available_mask);
1214 if (cpu >= nr_cpu_ids) /* empty */
1215 cpu = -1;
1216 else
1217 cpumask_set_cpu(cpu, &set->used);
1218
1219 mutex_unlock(&affinity->lock);
1220 hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
1221
1222 free_cpumask_var(intrs_mask);
1223free_available_mask:
1224 free_cpumask_var(available_mask);
1225free_hw_thread_mask:
1226 free_cpumask_var(hw_thread_mask);
1227free_diff:
1228 free_cpumask_var(diff);
1229done:
1230 return cpu;
1231}
1232
1233void hfi1_put_proc_affinity(int cpu)
1234{
1235 struct hfi1_affinity_node_list *affinity = &node_affinity;
1236 struct cpu_mask_set *set = &affinity->proc;
1237
1238 if (cpu < 0)
1239 return;
1240
1241 mutex_lock(&affinity->lock);
1242 cpu_mask_set_put(set, cpu);
1243 hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
1244 mutex_unlock(&affinity->lock);
1245}