1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Adjunct processor matrix VFIO device driver callbacks.
   4 *
   5 * Copyright IBM Corp. 2018
   6 *
   7 * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
   8 *	      Halil Pasic <pasic@linux.ibm.com>
   9 *	      Pierre Morel <pmorel@linux.ibm.com>
  10 */
  11#include <linux/string.h>
  12#include <linux/vfio.h>
  13#include <linux/device.h>
  14#include <linux/list.h>
  15#include <linux/ctype.h>
  16#include <linux/bitops.h>
  17#include <linux/kvm_host.h>
  18#include <linux/module.h>
  19#include <linux/uuid.h>
  20#include <asm/kvm.h>
  21#include <asm/zcrypt.h>
  22
  23#include "vfio_ap_private.h"
  24#include "vfio_ap_debug.h"
  25
  26#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
  27#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
  28
  29#define AP_QUEUE_ASSIGNED "assigned"
  30#define AP_QUEUE_UNASSIGNED "unassigned"
  31#define AP_QUEUE_IN_USE "in use"
  32
  33static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable);
  34static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
  35static const struct vfio_device_ops vfio_ap_matrix_dev_ops;
  36static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, unsigned int retry);
  37
  38/**
  39 * get_update_locks_for_kvm: Acquire the locks required to dynamically update a
  40 *			     KVM guest's APCB in the proper order.
  41 *
  42 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
  43 *
  44 * The proper locking order is:
  45 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
  46 *			       guest's APCB.
  47 * 2. kvm->lock:	       required to update a guest's APCB
  48 * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
  49 *
  50 * Note: If @kvm is NULL, the KVM lock will not be taken.
  51 */
  52static inline void get_update_locks_for_kvm(struct kvm *kvm)
  53{
  54	mutex_lock(&matrix_dev->guests_lock);
  55	if (kvm)
  56		mutex_lock(&kvm->lock);
  57	mutex_lock(&matrix_dev->mdevs_lock);
  58}
  59
  60/**
  61 * release_update_locks_for_kvm: Release the locks used to dynamically update a
  62 *				 KVM guest's APCB in the proper order.
  63 *
  64 * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
  65 *
  66 * The proper unlocking order is:
  67 * 1. matrix_dev->mdevs_lock
  68 * 2. kvm->lock
  69 * 3. matrix_dev->guests_lock
  70 *
  71 * Note: If @kvm is NULL, the KVM lock will not be released.
  72 */
  73static inline void release_update_locks_for_kvm(struct kvm *kvm)
  74{
  75	mutex_unlock(&matrix_dev->mdevs_lock);
  76	if (kvm)
  77		mutex_unlock(&kvm->lock);
  78	mutex_unlock(&matrix_dev->guests_lock);
  79}
  80
  81/**
  82 * get_update_locks_for_mdev: Acquire the locks required to dynamically update a
  83 *			      KVM guest's APCB in the proper order.
  84 *
  85 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
  86 *		 configuration data to use to update a KVM guest's APCB.
  87 *
  88 * The proper locking order is:
  89 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
  90 *			       guest's APCB.
  91 * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
  92 * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
  93 *
  94 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
  95 *	 lock will not be taken.
  96 */
  97static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
  98{
  99	mutex_lock(&matrix_dev->guests_lock);
 100	if (matrix_mdev && matrix_mdev->kvm)
 101		mutex_lock(&matrix_mdev->kvm->lock);
 102	mutex_lock(&matrix_dev->mdevs_lock);
 103}
 104
 105/**
 106 * release_update_locks_for_mdev: Release the locks used to dynamically update a
 107 *				  KVM guest's APCB in the proper order.
 108 *
 109 * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
 110 *		 configuration data to use to update a KVM guest's APCB.
 111 *
 112 * The proper unlocking order is:
 113 * 1. matrix_dev->mdevs_lock
 114 * 2. matrix_mdev->kvm->lock
 115 * 3. matrix_dev->guests_lock
 116 *
 117 * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
 118 *	 lock will not be released.
 119 */
 120static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
 121{
 122	mutex_unlock(&matrix_dev->mdevs_lock);
 123	if (matrix_mdev && matrix_mdev->kvm)
 124		mutex_unlock(&matrix_mdev->kvm->lock);
 125	mutex_unlock(&matrix_dev->guests_lock);
 126}
 127
 128/**
 129 * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and
 130 *			     acquire the locks required to update the APCB of
 131 *			     the KVM guest to which the mdev is attached.
 132 *
 133 * @apqn: the APQN of a queue device.
 134 *
 135 * The proper locking order is:
 136 * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
 137 *			       guest's APCB.
 138 * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
 139 * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
 140 *
 141 * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock
 142 *	 will not be taken.
 143 *
 144 * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn
 145 *	   is not assigned to an ap_matrix_mdev.
 146 */
 147static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn)
 148{
 149	struct ap_matrix_mdev *matrix_mdev;
 150
 151	mutex_lock(&matrix_dev->guests_lock);
 152
 153	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
 154		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&
 155		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {
 156			if (matrix_mdev->kvm)
 157				mutex_lock(&matrix_mdev->kvm->lock);
 158
 159			mutex_lock(&matrix_dev->mdevs_lock);
 160
 161			return matrix_mdev;
 162		}
 163	}
 164
 165	mutex_lock(&matrix_dev->mdevs_lock);
 166
 167	return NULL;
 168}
 169
 170/**
 171 * get_update_locks_for_queue: get the locks required to update the APCB of the
 172 *			       KVM guest to which the matrix mdev linked to a
 173 *			       vfio_ap_queue object is attached.
 174 *
 175 * @q: a pointer to a vfio_ap_queue object.
 176 *
 177 * The proper locking order is:
 178 * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a
 179 *				  KVM guest's APCB.
 180 * 2. q->matrix_mdev->kvm->lock:  required to update a guest's APCB
 181 * 3. matrix_dev->mdevs_lock:	  required to access data stored in matrix_mdev
 182 *
 183 * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock
 184 *	  will not be taken.
 185 */
 186static inline void get_update_locks_for_queue(struct vfio_ap_queue *q)
 187{
 188	mutex_lock(&matrix_dev->guests_lock);
 189	if (q->matrix_mdev && q->matrix_mdev->kvm)
 190		mutex_lock(&q->matrix_mdev->kvm->lock);
 191	mutex_lock(&matrix_dev->mdevs_lock);
 192}
 193
 194/**
 195 * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a
 196 *			    hash table of queues assigned to a matrix mdev
 197 * @matrix_mdev: the matrix mdev
 198 * @apqn: The APQN of a queue device
 199 *
 200 * Return: the pointer to the vfio_ap_queue struct representing the queue or
 201 *	   NULL if the queue is not assigned to @matrix_mdev
 202 */
 203static struct vfio_ap_queue *vfio_ap_mdev_get_queue(
 204					struct ap_matrix_mdev *matrix_mdev,
 205					int apqn)
 206{
 207	struct vfio_ap_queue *q;
 208
 209	hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode,
 210			       apqn) {
 211		if (q && q->apqn == apqn)
 212			return q;
 213	}
 214
 215	return NULL;
 216}
 217
 218/**
 219 * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries
 220 * @apqn: The AP Queue number
 221 *
 222 * Checks the IRQ bit for the status of this APQN using ap_tapq.
 223 * Returns if the ap_tapq function succeeded and the bit is clear.
 224 * Returns if ap_tapq function failed with invalid, deconfigured or
 225 * checkstopped AP.
 226 * Otherwise retries up to 5 times after waiting 20ms.
 227 */
 228static void vfio_ap_wait_for_irqclear(int apqn)
 229{
 230	struct ap_queue_status status;
 231	int retry = 5;
 232
 233	do {
 234		status = ap_tapq(apqn, NULL);
 235		switch (status.response_code) {
 236		case AP_RESPONSE_NORMAL:
 237		case AP_RESPONSE_RESET_IN_PROGRESS:
 238			if (!status.irq_enabled)
 239				return;
 240			fallthrough;
 241		case AP_RESPONSE_BUSY:
 242			msleep(20);
 243			break;
 244		case AP_RESPONSE_Q_NOT_AVAIL:
 245		case AP_RESPONSE_DECONFIGURED:
 246		case AP_RESPONSE_CHECKSTOPPED:
 247		default:
 248			WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__,
 249				  status.response_code, apqn);
 250			return;
 251		}
 252	} while (--retry);
 253
 254	WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n",
 255		  __func__, status.response_code, apqn);
 256}
 257
 258/**
 259 * vfio_ap_free_aqic_resources - free vfio_ap_queue resources
 260 * @q: The vfio_ap_queue
 261 *
 262 * Unregisters the ISC in the GIB when the saved ISC not invalid.
 263 * Unpins the guest's page holding the NIB when it exists.
 264 * Resets the saved_iova and saved_isc to invalid values.
 265 */
 266static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)
 267{
 268	if (!q)
 269		return;
 270	if (q->saved_isc != VFIO_AP_ISC_INVALID &&
 271	    !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {
 272		kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
 273		q->saved_isc = VFIO_AP_ISC_INVALID;
 274	}
 275	if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) {
 276		vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1);
 277		q->saved_iova = 0;
 278	}
 279}
 280
 281/**
 282 * vfio_ap_irq_disable - disables and clears an ap_queue interrupt
 283 * @q: The vfio_ap_queue
 284 *
 285 * Uses ap_aqic to disable the interruption and in case of success, reset
 286 * in progress or IRQ disable command already proceeded: calls
 287 * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear
 288 * and calls vfio_ap_free_aqic_resources() to free the resources associated
 289 * with the AP interrupt handling.
 290 *
 291 * In the case the AP is busy, or a reset is in progress,
 292 * retries after 20ms, up to 5 times.
 293 *
 294 * Returns if ap_aqic function failed with invalid, deconfigured or
 295 * checkstopped AP.
 296 *
 297 * Return: &struct ap_queue_status
 298 */
 299static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q)
 300{
 301	struct ap_qirq_ctrl aqic_gisa = {};
 302	struct ap_queue_status status;
 303	int retries = 5;
 304
 305	do {
 306		status = ap_aqic(q->apqn, aqic_gisa, 0);
 307		switch (status.response_code) {
 308		case AP_RESPONSE_OTHERWISE_CHANGED:
 309		case AP_RESPONSE_NORMAL:
 310			vfio_ap_wait_for_irqclear(q->apqn);
 311			goto end_free;
 312		case AP_RESPONSE_RESET_IN_PROGRESS:
 313		case AP_RESPONSE_BUSY:
 314			msleep(20);
 315			break;
 316		case AP_RESPONSE_Q_NOT_AVAIL:
 317		case AP_RESPONSE_DECONFIGURED:
 318		case AP_RESPONSE_CHECKSTOPPED:
 319		case AP_RESPONSE_INVALID_ADDRESS:
 320		default:
 321			/* All cases in default means AP not operational */
 322			WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
 323				  status.response_code);
 324			goto end_free;
 325		}
 326	} while (retries--);
 327
 328	WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
 329		  status.response_code);
 330end_free:
 331	vfio_ap_free_aqic_resources(q);
 332	return status;
 333}
 334
 335/**
 336 * vfio_ap_validate_nib - validate a notification indicator byte (nib) address.
 337 *
 338 * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction.
 339 * @nib: the location for storing the nib address.
 340 *
 341 * When the PQAP(AQIC) instruction is executed, general register 2 contains the
 342 * address of the notification indicator byte (nib) used for IRQ notification.
 343 * This function parses and validates the nib from gr2.
 344 *
 345 * Return: returns zero if the nib address is a valid; otherwise, returns
 346 *	   -EINVAL.
 347 */
 348static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib)
 349{
 350	*nib = vcpu->run->s.regs.gprs[2];
 351
 352	if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT)))
 353		return -EINVAL;
 354
 355	return 0;
 356}
 357
 358/**
 359 * vfio_ap_irq_enable - Enable Interruption for a APQN
 360 *
 361 * @q:	 the vfio_ap_queue holding AQIC parameters
 362 * @isc: the guest ISC to register with the GIB interface
 363 * @vcpu: the vcpu object containing the registers specifying the parameters
 364 *	  passed to the PQAP(AQIC) instruction.
 365 *
 366 * Pin the NIB saved in *q
 367 * Register the guest ISC to GIB interface and retrieve the
 368 * host ISC to issue the host side PQAP/AQIC
 369 *
 370 * Response.status may be set to AP_RESPONSE_INVALID_ADDRESS in case the
 371 * vfio_pin_pages failed.
 372 *
 373 * Otherwise return the ap_queue_status returned by the ap_aqic(),
 374 * all retry handling will be done by the guest.
 375 *
 376 * Return: &struct ap_queue_status
 377 */
 378static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q,
 379						 int isc,
 380						 struct kvm_vcpu *vcpu)
 381{
 382	struct ap_qirq_ctrl aqic_gisa = {};
 383	struct ap_queue_status status = {};
 384	struct kvm_s390_gisa *gisa;
 385	struct page *h_page;
 386	int nisc;
 387	struct kvm *kvm;
 388	phys_addr_t h_nib;
 389	dma_addr_t nib;
 390	int ret;
 391
 392	/* Verify that the notification indicator byte address is valid */
 393	if (vfio_ap_validate_nib(vcpu, &nib)) {
 394		VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n",
 395				 __func__, &nib, q->apqn);
 396
 397		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
 398		return status;
 399	}
 400
 401	ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1,
 402			     IOMMU_READ | IOMMU_WRITE, &h_page);
 403	switch (ret) {
 404	case 1:
 405		break;
 406	default:
 407		VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d,"
 408				 "nib=%pad, apqn=%#04x\n",
 409				 __func__, ret, &nib, q->apqn);
 410
 411		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
 412		return status;
 413	}
 414
 415	kvm = q->matrix_mdev->kvm;
 416	gisa = kvm->arch.gisa_int.origin;
 417
 418	h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK);
 419	aqic_gisa.gisc = isc;
 420
 421	nisc = kvm_s390_gisc_register(kvm, isc);
 422	if (nisc < 0) {
 423		VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n",
 424				 __func__, nisc, isc, q->apqn);
 425
 426		status.response_code = AP_RESPONSE_INVALID_GISA;
 427		return status;
 428	}
 429
 430	aqic_gisa.isc = nisc;
 431	aqic_gisa.ir = 1;
 432	aqic_gisa.gisa = virt_to_phys(gisa) >> 4;
 433
 434	status = ap_aqic(q->apqn, aqic_gisa, h_nib);
 435	switch (status.response_code) {
 436	case AP_RESPONSE_NORMAL:
 437		/* See if we did clear older IRQ configuration */
 438		vfio_ap_free_aqic_resources(q);
 439		q->saved_iova = nib;
 440		q->saved_isc = isc;
 441		break;
 442	case AP_RESPONSE_OTHERWISE_CHANGED:
 443		/* We could not modify IRQ setings: clear new configuration */
 444		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
 445		kvm_s390_gisc_unregister(kvm, isc);
 446		break;
 447	default:
 448		pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn,
 449			status.response_code);
 450		vfio_ap_irq_disable(q);
 451		break;
 452	}
 453
 454	if (status.response_code != AP_RESPONSE_NORMAL) {
 455		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: "
 456				 "zone=%#x, ir=%#x, gisc=%#x, f=%#x,"
 457				 "gisa=%#x, isc=%#x, apqn=%#04x\n",
 458				 __func__, status.response_code,
 459				 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc,
 460				 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc,
 461				 q->apqn);
 462	}
 463
 464	return status;
 465}
 466
 467/**
 468 * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array
 469 *				of big endian elements that can be passed by
 470 *				value to an s390dbf sprintf event function to
 471 *				format a UUID string.
 472 *
 473 * @guid: the object containing the little endian guid
 474 * @uuid: a six-element array of long values that can be passed by value as
 475 *	  arguments for a formatting string specifying a UUID.
 476 *
 477 * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf
 478 * event functions if the memory for the passed string is available as long as
 479 * the debug feature exists. Since a mediated device can be removed at any
 480 * time, it's name can not be used because %s passes the reference to the string
 481 * in memory and the reference will go stale once the device is removed .
 482 *
 483 * The s390dbf string formatting function allows a maximum of 9 arguments for a
 484 * message to be displayed in the 'sprintf' view. In order to use the bytes
 485 * comprising the mediated device's UUID to display the mediated device name,
 486 * they will have to be converted into an array whose elements can be passed by
 487 * value to sprintf. For example:
 488 *
 489 * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 }
 490 * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804
 491 * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 }
 492 * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx"
 493 */
 494static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid)
 495{
 496	/*
 497	 * The input guid is ordered in little endian, so it needs to be
 498	 * reordered for displaying a UUID as a string. This specifies the
 499	 * guid indices in proper order.
 500	 */
 501	uuid[0] = le32_to_cpup((__le32 *)guid);
 502	uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]);
 503	uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]);
 504	uuid[3] = *((__u16 *)&guid->b[8]);
 505	uuid[4] = *((__u16 *)&guid->b[10]);
 506	uuid[5] = *((__u32 *)&guid->b[12]);
 507}
 508
 509/**
 510 * handle_pqap - PQAP instruction callback
 511 *
 512 * @vcpu: The vcpu on which we received the PQAP instruction
 513 *
 514 * Get the general register contents to initialize internal variables.
 515 * REG[0]: APQN
 516 * REG[1]: IR and ISC
 517 * REG[2]: NIB
 518 *
 519 * Response.status may be set to following Response Code:
 520 * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available
 521 * - AP_RESPONSE_DECONFIGURED: if the queue is not configured
 522 * - AP_RESPONSE_NORMAL (0) : in case of successs
 523 *   Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC.
 524 * We take the matrix_dev lock to ensure serialization on queues and
 525 * mediated device access.
 526 *
 527 * Return: 0 if we could handle the request inside KVM.
 528 * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault.
 529 */
 530static int handle_pqap(struct kvm_vcpu *vcpu)
 531{
 532	uint64_t status;
 533	uint16_t apqn;
 534	unsigned long uuid[6];
 535	struct vfio_ap_queue *q;
 536	struct ap_queue_status qstatus = {
 537			       .response_code = AP_RESPONSE_Q_NOT_AVAIL, };
 538	struct ap_matrix_mdev *matrix_mdev;
 539
 540	apqn = vcpu->run->s.regs.gprs[0] & 0xffff;
 541
 542	/* If we do not use the AIV facility just go to userland */
 543	if (!(vcpu->arch.sie_block->eca & ECA_AIV)) {
 544		VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n",
 545				 __func__, apqn, vcpu->arch.sie_block->eca);
 546
 547		return -EOPNOTSUPP;
 548	}
 549
 550	mutex_lock(&matrix_dev->mdevs_lock);
 551
 552	if (!vcpu->kvm->arch.crypto.pqap_hook) {
 553		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n",
 554				 __func__, apqn);
 555
 556		goto out_unlock;
 557	}
 558
 559	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
 560				   struct ap_matrix_mdev, pqap_hook);
 561
 562	/* If the there is no guest using the mdev, there is nothing to do */
 563	if (!matrix_mdev->kvm) {
 564		vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid);
 565		VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n",
 566				 __func__, uuid[0],  uuid[1], uuid[2],
 567				 uuid[3], uuid[4], uuid[5], apqn);
 568		goto out_unlock;
 569	}
 570
 571	q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
 572	if (!q) {
 573		VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n",
 574				 __func__, AP_QID_CARD(apqn),
 575				 AP_QID_QUEUE(apqn));
 576		goto out_unlock;
 577	}
 578
 579	status = vcpu->run->s.regs.gprs[1];
 580
 581	/* If IR bit(16) is set we enable the interrupt */
 582	if ((status >> (63 - 16)) & 0x01)
 583		qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu);
 584	else
 585		qstatus = vfio_ap_irq_disable(q);
 586
 587out_unlock:
 588	memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus));
 589	vcpu->run->s.regs.gprs[1] >>= 32;
 590	mutex_unlock(&matrix_dev->mdevs_lock);
 591	return 0;
 592}
 593
 594static void vfio_ap_matrix_init(struct ap_config_info *info,
 595				struct ap_matrix *matrix)
 596{
 597	matrix->apm_max = info->apxa ? info->Na : 63;
 598	matrix->aqm_max = info->apxa ? info->Nd : 15;
 599	matrix->adm_max = info->apxa ? info->Nd : 15;
 600}
 601
 602static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev)
 603{
 604	if (matrix_mdev->kvm)
 605		kvm_arch_crypto_set_masks(matrix_mdev->kvm,
 606					  matrix_mdev->shadow_apcb.apm,
 607					  matrix_mdev->shadow_apcb.aqm,
 608					  matrix_mdev->shadow_apcb.adm);
 609}
 610
 611static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev)
 612{
 613	DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS);
 614
 615	bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS);
 616	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,
 617		   (unsigned long *)matrix_dev->info.adm, AP_DOMAINS);
 618
 619	return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm,
 620			     AP_DOMAINS);
 621}
 622
 623/*
 624 * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
 625 *				to ensure no queue devices are passed through to
 626 *				the guest that are not bound to the vfio_ap
 627 *				device driver.
 628 *
 629 * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
 630 *
 631 * Note: If an APQN referencing a queue device that is not bound to the vfio_ap
 632 *	 driver, its APID will be filtered from the guest's APCB. The matrix
 633 *	 structure precludes filtering an individual APQN, so its APID will be
 634 *	 filtered.
 635 *
 636 * Return: a boolean value indicating whether the KVM guest's APCB was changed
 637 *	   by the filtering or not.
 638 */
 639static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm,
 640				       struct ap_matrix_mdev *matrix_mdev)
 641{
 642	unsigned long apid, apqi, apqn;
 643	DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES);
 644	DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS);
 645	struct vfio_ap_queue *q;
 646
 647	bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES);
 648	bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS);
 649	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
 650
 651	/*
 652	 * Copy the adapters, domains and control domains to the shadow_apcb
 653	 * from the matrix mdev, but only those that are assigned to the host's
 654	 * AP configuration.
 655	 */
 656	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,
 657		   (unsigned long *)matrix_dev->info.apm, AP_DEVICES);
 658	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,
 659		   (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS);
 660
 661	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
 662		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
 663			/*
 664			 * If the APQN is not bound to the vfio_ap device
 665			 * driver, then we can't assign it to the guest's
 666			 * AP configuration. The AP architecture won't
 667			 * allow filtering of a single APQN, so let's filter
 668			 * the APID since an adapter represents a physical
 669			 * hardware device.
 670			 */
 671			apqn = AP_MKQID(apid, apqi);
 672			q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
 673			if (!q || q->reset_rc) {
 674				clear_bit_inv(apid,
 675					      matrix_mdev->shadow_apcb.apm);
 676				break;
 677			}
 678		}
 679	}
 680
 681	return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm,
 682			     AP_DEVICES) ||
 683	       !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm,
 684			     AP_DOMAINS);
 685}
 686
 687static int vfio_ap_mdev_init_dev(struct vfio_device *vdev)
 688{
 689	struct ap_matrix_mdev *matrix_mdev =
 690		container_of(vdev, struct ap_matrix_mdev, vdev);
 691
 692	matrix_mdev->mdev = to_mdev_device(vdev->dev);
 693	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
 694	matrix_mdev->pqap_hook = handle_pqap;
 695	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
 696	hash_init(matrix_mdev->qtable.queues);
 697
 698	return 0;
 699}
 700
 701static int vfio_ap_mdev_probe(struct mdev_device *mdev)
 702{
 703	struct ap_matrix_mdev *matrix_mdev;
 704	int ret;
 705
 706	matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev,
 707					&vfio_ap_matrix_dev_ops);
 708	if (IS_ERR(matrix_mdev))
 709		return PTR_ERR(matrix_mdev);
 710
 711	ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev);
 712	if (ret)
 713		goto err_put_vdev;
 714	dev_set_drvdata(&mdev->dev, matrix_mdev);
 715	mutex_lock(&matrix_dev->mdevs_lock);
 716	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
 717	mutex_unlock(&matrix_dev->mdevs_lock);
 718	return 0;
 719
 720err_put_vdev:
 721	vfio_put_device(&matrix_mdev->vdev);
 722	return ret;
 723}
 724
 725static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev,
 726				    struct vfio_ap_queue *q)
 727{
 728	if (q) {
 729		q->matrix_mdev = matrix_mdev;
 730		hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn);
 731	}
 732}
 733
 734static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn)
 735{
 736	struct vfio_ap_queue *q;
 737
 738	q = vfio_ap_find_queue(apqn);
 739	vfio_ap_mdev_link_queue(matrix_mdev, q);
 740}
 741
 742static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q)
 743{
 744	hash_del(&q->mdev_qnode);
 745}
 746
 747static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q)
 748{
 749	q->matrix_mdev = NULL;
 750}
 751
 752static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev)
 753{
 754	struct vfio_ap_queue *q;
 755	unsigned long apid, apqi;
 756
 757	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
 758		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
 759				     AP_DOMAINS) {
 760			q = vfio_ap_mdev_get_queue(matrix_mdev,
 761						   AP_MKQID(apid, apqi));
 762			if (q)
 763				q->matrix_mdev = NULL;
 764		}
 765	}
 766}
 767
 768static void vfio_ap_mdev_remove(struct mdev_device *mdev)
 769{
 770	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
 771
 772	vfio_unregister_group_dev(&matrix_mdev->vdev);
 773
 774	mutex_lock(&matrix_dev->guests_lock);
 775	mutex_lock(&matrix_dev->mdevs_lock);
 776	vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
 777	vfio_ap_mdev_unlink_fr_queues(matrix_mdev);
 778	list_del(&matrix_mdev->node);
 779	mutex_unlock(&matrix_dev->mdevs_lock);
 780	mutex_unlock(&matrix_dev->guests_lock);
 781	vfio_put_device(&matrix_mdev->vdev);
 782}
 783
 784#define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \
 785			 "already assigned to %s"
 786
 787static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev,
 788					 unsigned long *apm,
 789					 unsigned long *aqm)
 790{
 791	unsigned long apid, apqi;
 792	const struct device *dev = mdev_dev(matrix_mdev->mdev);
 793	const char *mdev_name = dev_name(dev);
 794
 795	for_each_set_bit_inv(apid, apm, AP_DEVICES)
 796		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS)
 797			dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name);
 798}
 799
 800/**
 801 * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
 802 *
 803 * @mdev_apm: mask indicating the APIDs of the APQNs to be verified
 804 * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified
 805 *
 806 * Verifies that each APQN derived from the Cartesian product of a bitmap of
 807 * AP adapter IDs and AP queue indexes is not configured for any matrix
 808 * mediated device. AP queue sharing is not allowed.
 809 *
 810 * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE.
 811 */
 812static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm,
 813					  unsigned long *mdev_aqm)
 814{
 815	struct ap_matrix_mdev *matrix_mdev;
 816	DECLARE_BITMAP(apm, AP_DEVICES);
 817	DECLARE_BITMAP(aqm, AP_DOMAINS);
 818
 819	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
 820		/*
 821		 * If the input apm and aqm are fields of the matrix_mdev
 822		 * object, then move on to the next matrix_mdev.
 823		 */
 824		if (mdev_apm == matrix_mdev->matrix.apm &&
 825		    mdev_aqm == matrix_mdev->matrix.aqm)
 826			continue;
 827
 828		memset(apm, 0, sizeof(apm));
 829		memset(aqm, 0, sizeof(aqm));
 830
 831		/*
 832		 * We work on full longs, as we can only exclude the leftover
 833		 * bits in non-inverse order. The leftover is all zeros.
 834		 */
 835		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,
 836				AP_DEVICES))
 837			continue;
 838
 839		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,
 840				AP_DOMAINS))
 841			continue;
 842
 843		vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm);
 844
 845		return -EADDRINUSE;
 846	}
 847
 848	return 0;
 849}
 850
 851/**
 852 * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are
 853 *				 not reserved for the default zcrypt driver and
 854 *				 are not assigned to another mdev.
 855 *
 856 * @matrix_mdev: the mdev to which the APQNs being validated are assigned.
 857 *
 858 * Return: One of the following values:
 859 * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function,
 860 *   most likely -EBUSY indicating the ap_perms_mutex lock is already held.
 861 * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the
 862 *		   zcrypt default driver.
 863 * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev
 864 * o A zero indicating validation succeeded.
 865 */
 866static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev)
 867{
 868	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,
 869					       matrix_mdev->matrix.aqm))
 870		return -EADDRNOTAVAIL;
 871
 872	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,
 873					      matrix_mdev->matrix.aqm);
 874}
 875
 876static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev,
 877				      unsigned long apid)
 878{
 879	unsigned long apqi;
 880
 881	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)
 882		vfio_ap_mdev_link_apqn(matrix_mdev,
 883				       AP_MKQID(apid, apqi));
 884}
 885
 886/**
 887 * assign_adapter_store - parses the APID from @buf and sets the
 888 * corresponding bit in the mediated matrix device's APM
 889 *
 890 * @dev:	the matrix device
 891 * @attr:	the mediated matrix device's assign_adapter attribute
 892 * @buf:	a buffer containing the AP adapter number (APID) to
 893 *		be assigned
 894 * @count:	the number of bytes in @buf
 895 *
 896 * Return: the number of bytes processed if the APID is valid; otherwise,
 897 * returns one of the following errors:
 898 *
 899 *	1. -EINVAL
 900 *	   The APID is not a valid number
 901 *
 902 *	2. -ENODEV
 903 *	   The APID exceeds the maximum value configured for the system
 904 *
 905 *	3. -EADDRNOTAVAIL
 906 *	   An APQN derived from the cross product of the APID being assigned
 907 *	   and the APQIs previously assigned is not bound to the vfio_ap device
 908 *	   driver; or, if no APQIs have yet been assigned, the APID is not
 909 *	   contained in an APQN bound to the vfio_ap device driver.
 910 *
 911 *	4. -EADDRINUSE
 912 *	   An APQN derived from the cross product of the APID being assigned
 913 *	   and the APQIs previously assigned is being used by another mediated
 914 *	   matrix device
 915 *
 916 *	5. -EAGAIN
 917 *	   A lock required to validate the mdev's AP configuration could not
 918 *	   be obtained.
 919 */
 920static ssize_t assign_adapter_store(struct device *dev,
 921				    struct device_attribute *attr,
 922				    const char *buf, size_t count)
 923{
 924	int ret;
 925	unsigned long apid;
 926	DECLARE_BITMAP(apm_delta, AP_DEVICES);
 927	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
 928
 929	mutex_lock(&ap_perms_mutex);
 930	get_update_locks_for_mdev(matrix_mdev);
 931
 932	ret = kstrtoul(buf, 0, &apid);
 933	if (ret)
 934		goto done;
 935
 936	if (apid > matrix_mdev->matrix.apm_max) {
 937		ret = -ENODEV;
 938		goto done;
 939	}
 940
 941	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {
 942		ret = count;
 943		goto done;
 944	}
 945
 946	set_bit_inv(apid, matrix_mdev->matrix.apm);
 947
 948	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
 949	if (ret) {
 950		clear_bit_inv(apid, matrix_mdev->matrix.apm);
 951		goto done;
 952	}
 953
 954	vfio_ap_mdev_link_adapter(matrix_mdev, apid);
 955	memset(apm_delta, 0, sizeof(apm_delta));
 956	set_bit_inv(apid, apm_delta);
 957
 958	if (vfio_ap_mdev_filter_matrix(apm_delta,
 959				       matrix_mdev->matrix.aqm, matrix_mdev))
 960		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
 961
 962	ret = count;
 963done:
 964	release_update_locks_for_mdev(matrix_mdev);
 965	mutex_unlock(&ap_perms_mutex);
 966
 967	return ret;
 968}
 969static DEVICE_ATTR_WO(assign_adapter);
 970
 971static struct vfio_ap_queue
 972*vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev,
 973			     unsigned long apid, unsigned long apqi)
 974{
 975	struct vfio_ap_queue *q = NULL;
 976
 977	q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
 978	/* If the queue is assigned to the matrix mdev, unlink it. */
 979	if (q)
 980		vfio_ap_unlink_queue_fr_mdev(q);
 981
 982	return q;
 983}
 984
 985/**
 986 * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned
 987 *				 adapter from the matrix mdev to which the
 988 *				 adapter was assigned.
 989 * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
 990 * @apid: the APID of the unassigned adapter.
 991 * @qtable: table for storing queues associated with unassigned adapter.
 992 */
 993static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev,
 994					unsigned long apid,
 995					struct ap_queue_table *qtable)
 996{
 997	unsigned long apqi;
 998	struct vfio_ap_queue *q;
 999
1000	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {
1001		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1002
1003		if (q && qtable) {
1004			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1005			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
1006				hash_add(qtable->queues, &q->mdev_qnode,
1007					 q->apqn);
1008		}
1009	}
1010}
1011
1012static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev,
1013					    unsigned long apid)
1014{
1015	int loop_cursor;
1016	struct vfio_ap_queue *q;
1017	struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL);
1018
1019	hash_init(qtable->queues);
1020	vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, qtable);
1021
1022	if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) {
1023		clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
1024		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1025	}
1026
1027	vfio_ap_mdev_reset_queues(qtable);
1028
1029	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1030		vfio_ap_unlink_mdev_fr_queue(q);
1031		hash_del(&q->mdev_qnode);
1032	}
1033
1034	kfree(qtable);
1035}
1036
1037/**
1038 * unassign_adapter_store - parses the APID from @buf and clears the
1039 * corresponding bit in the mediated matrix device's APM
1040 *
1041 * @dev:	the matrix device
1042 * @attr:	the mediated matrix device's unassign_adapter attribute
1043 * @buf:	a buffer containing the adapter number (APID) to be unassigned
1044 * @count:	the number of bytes in @buf
1045 *
1046 * Return: the number of bytes processed if the APID is valid; otherwise,
1047 * returns one of the following errors:
1048 *	-EINVAL if the APID is not a number
1049 *	-ENODEV if the APID it exceeds the maximum value configured for the
1050 *		system
1051 */
1052static ssize_t unassign_adapter_store(struct device *dev,
1053				      struct device_attribute *attr,
1054				      const char *buf, size_t count)
1055{
1056	int ret;
1057	unsigned long apid;
1058	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1059
1060	get_update_locks_for_mdev(matrix_mdev);
1061
1062	ret = kstrtoul(buf, 0, &apid);
1063	if (ret)
1064		goto done;
1065
1066	if (apid > matrix_mdev->matrix.apm_max) {
1067		ret = -ENODEV;
1068		goto done;
1069	}
1070
1071	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {
1072		ret = count;
1073		goto done;
1074	}
1075
1076	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
1077	vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid);
1078	ret = count;
1079done:
1080	release_update_locks_for_mdev(matrix_mdev);
1081	return ret;
1082}
1083static DEVICE_ATTR_WO(unassign_adapter);
1084
1085static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev,
1086				     unsigned long apqi)
1087{
1088	unsigned long apid;
1089
1090	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)
1091		vfio_ap_mdev_link_apqn(matrix_mdev,
1092				       AP_MKQID(apid, apqi));
1093}
1094
1095/**
1096 * assign_domain_store - parses the APQI from @buf and sets the
1097 * corresponding bit in the mediated matrix device's AQM
1098 *
1099 * @dev:	the matrix device
1100 * @attr:	the mediated matrix device's assign_domain attribute
1101 * @buf:	a buffer containing the AP queue index (APQI) of the domain to
1102 *		be assigned
1103 * @count:	the number of bytes in @buf
1104 *
1105 * Return: the number of bytes processed if the APQI is valid; otherwise returns
1106 * one of the following errors:
1107 *
1108 *	1. -EINVAL
1109 *	   The APQI is not a valid number
1110 *
1111 *	2. -ENODEV
1112 *	   The APQI exceeds the maximum value configured for the system
1113 *
1114 *	3. -EADDRNOTAVAIL
1115 *	   An APQN derived from the cross product of the APQI being assigned
1116 *	   and the APIDs previously assigned is not bound to the vfio_ap device
1117 *	   driver; or, if no APIDs have yet been assigned, the APQI is not
1118 *	   contained in an APQN bound to the vfio_ap device driver.
1119 *
1120 *	4. -EADDRINUSE
1121 *	   An APQN derived from the cross product of the APQI being assigned
1122 *	   and the APIDs previously assigned is being used by another mediated
1123 *	   matrix device
1124 *
1125 *	5. -EAGAIN
1126 *	   The lock required to validate the mdev's AP configuration could not
1127 *	   be obtained.
1128 */
1129static ssize_t assign_domain_store(struct device *dev,
1130				   struct device_attribute *attr,
1131				   const char *buf, size_t count)
1132{
1133	int ret;
1134	unsigned long apqi;
1135	DECLARE_BITMAP(aqm_delta, AP_DOMAINS);
1136	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1137
1138	mutex_lock(&ap_perms_mutex);
1139	get_update_locks_for_mdev(matrix_mdev);
1140
1141	ret = kstrtoul(buf, 0, &apqi);
1142	if (ret)
1143		goto done;
1144
1145	if (apqi > matrix_mdev->matrix.aqm_max) {
1146		ret = -ENODEV;
1147		goto done;
1148	}
1149
1150	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1151		ret = count;
1152		goto done;
1153	}
1154
1155	set_bit_inv(apqi, matrix_mdev->matrix.aqm);
1156
1157	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
1158	if (ret) {
1159		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);
1160		goto done;
1161	}
1162
1163	vfio_ap_mdev_link_domain(matrix_mdev, apqi);
1164	memset(aqm_delta, 0, sizeof(aqm_delta));
1165	set_bit_inv(apqi, aqm_delta);
1166
1167	if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta,
1168				       matrix_mdev))
1169		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1170
1171	ret = count;
1172done:
1173	release_update_locks_for_mdev(matrix_mdev);
1174	mutex_unlock(&ap_perms_mutex);
1175
1176	return ret;
1177}
1178static DEVICE_ATTR_WO(assign_domain);
1179
1180static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev,
1181				       unsigned long apqi,
1182				       struct ap_queue_table *qtable)
1183{
1184	unsigned long apid;
1185	struct vfio_ap_queue *q;
1186
1187	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
1188		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1189
1190		if (q && qtable) {
1191			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1192			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
1193				hash_add(qtable->queues, &q->mdev_qnode,
1194					 q->apqn);
1195		}
1196	}
1197}
1198
1199static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev,
1200					   unsigned long apqi)
1201{
1202	int loop_cursor;
1203	struct vfio_ap_queue *q;
1204	struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL);
1205
1206	hash_init(qtable->queues);
1207	vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, qtable);
1208
1209	if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
1210		clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm);
1211		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1212	}
1213
1214	vfio_ap_mdev_reset_queues(qtable);
1215
1216	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1217		vfio_ap_unlink_mdev_fr_queue(q);
1218		hash_del(&q->mdev_qnode);
1219	}
1220
1221	kfree(qtable);
1222}
1223
1224/**
1225 * unassign_domain_store - parses the APQI from @buf and clears the
1226 * corresponding bit in the mediated matrix device's AQM
1227 *
1228 * @dev:	the matrix device
1229 * @attr:	the mediated matrix device's unassign_domain attribute
1230 * @buf:	a buffer containing the AP queue index (APQI) of the domain to
1231 *		be unassigned
1232 * @count:	the number of bytes in @buf
1233 *
1234 * Return: the number of bytes processed if the APQI is valid; otherwise,
1235 * returns one of the following errors:
1236 *	-EINVAL if the APQI is not a number
1237 *	-ENODEV if the APQI exceeds the maximum value configured for the system
1238 */
1239static ssize_t unassign_domain_store(struct device *dev,
1240				     struct device_attribute *attr,
1241				     const char *buf, size_t count)
1242{
1243	int ret;
1244	unsigned long apqi;
1245	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1246
1247	get_update_locks_for_mdev(matrix_mdev);
1248
1249	ret = kstrtoul(buf, 0, &apqi);
1250	if (ret)
1251		goto done;
1252
1253	if (apqi > matrix_mdev->matrix.aqm_max) {
1254		ret = -ENODEV;
1255		goto done;
1256	}
1257
1258	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1259		ret = count;
1260		goto done;
1261	}
1262
1263	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
1264	vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi);
1265	ret = count;
1266
1267done:
1268	release_update_locks_for_mdev(matrix_mdev);
1269	return ret;
1270}
1271static DEVICE_ATTR_WO(unassign_domain);
1272
1273/**
1274 * assign_control_domain_store - parses the domain ID from @buf and sets
1275 * the corresponding bit in the mediated matrix device's ADM
1276 *
1277 * @dev:	the matrix device
1278 * @attr:	the mediated matrix device's assign_control_domain attribute
1279 * @buf:	a buffer containing the domain ID to be assigned
1280 * @count:	the number of bytes in @buf
1281 *
1282 * Return: the number of bytes processed if the domain ID is valid; otherwise,
1283 * returns one of the following errors:
1284 *	-EINVAL if the ID is not a number
1285 *	-ENODEV if the ID exceeds the maximum value configured for the system
1286 */
1287static ssize_t assign_control_domain_store(struct device *dev,
1288					   struct device_attribute *attr,
1289					   const char *buf, size_t count)
1290{
1291	int ret;
1292	unsigned long id;
1293	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1294
1295	get_update_locks_for_mdev(matrix_mdev);
1296
1297	ret = kstrtoul(buf, 0, &id);
1298	if (ret)
1299		goto done;
1300
1301	if (id > matrix_mdev->matrix.adm_max) {
1302		ret = -ENODEV;
1303		goto done;
1304	}
1305
1306	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {
1307		ret = count;
1308		goto done;
1309	}
1310
1311	/* Set the bit in the ADM (bitmask) corresponding to the AP control
1312	 * domain number (id). The bits in the mask, from most significant to
1313	 * least significant, correspond to IDs 0 up to the one less than the
1314	 * number of control domains that can be assigned.
1315	 */
1316	set_bit_inv(id, matrix_mdev->matrix.adm);
1317	if (vfio_ap_mdev_filter_cdoms(matrix_mdev))
1318		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1319
1320	ret = count;
1321done:
1322	release_update_locks_for_mdev(matrix_mdev);
1323	return ret;
1324}
1325static DEVICE_ATTR_WO(assign_control_domain);
1326
1327/**
1328 * unassign_control_domain_store - parses the domain ID from @buf and
1329 * clears the corresponding bit in the mediated matrix device's ADM
1330 *
1331 * @dev:	the matrix device
1332 * @attr:	the mediated matrix device's unassign_control_domain attribute
1333 * @buf:	a buffer containing the domain ID to be unassigned
1334 * @count:	the number of bytes in @buf
1335 *
1336 * Return: the number of bytes processed if the domain ID is valid; otherwise,
1337 * returns one of the following errors:
1338 *	-EINVAL if the ID is not a number
1339 *	-ENODEV if the ID exceeds the maximum value configured for the system
1340 */
1341static ssize_t unassign_control_domain_store(struct device *dev,
1342					     struct device_attribute *attr,
1343					     const char *buf, size_t count)
1344{
1345	int ret;
1346	unsigned long domid;
1347	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1348
1349	get_update_locks_for_mdev(matrix_mdev);
1350
1351	ret = kstrtoul(buf, 0, &domid);
1352	if (ret)
1353		goto done;
1354
1355	if (domid > matrix_mdev->matrix.adm_max) {
1356		ret = -ENODEV;
1357		goto done;
1358	}
1359
1360	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {
1361		ret = count;
1362		goto done;
1363	}
1364
1365	clear_bit_inv(domid, matrix_mdev->matrix.adm);
1366
1367	if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) {
1368		clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm);
1369		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1370	}
1371
1372	ret = count;
1373done:
1374	release_update_locks_for_mdev(matrix_mdev);
1375	return ret;
1376}
1377static DEVICE_ATTR_WO(unassign_control_domain);
1378
1379static ssize_t control_domains_show(struct device *dev,
1380				    struct device_attribute *dev_attr,
1381				    char *buf)
1382{
1383	unsigned long id;
1384	int nchars = 0;
1385	int n;
1386	char *bufpos = buf;
1387	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1388	unsigned long max_domid = matrix_mdev->matrix.adm_max;
1389
1390	mutex_lock(&matrix_dev->mdevs_lock);
1391	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {
1392		n = sprintf(bufpos, "%04lx\n", id);
1393		bufpos += n;
1394		nchars += n;
1395	}
1396	mutex_unlock(&matrix_dev->mdevs_lock);
1397
1398	return nchars;
1399}
1400static DEVICE_ATTR_RO(control_domains);
1401
1402static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)
1403{
1404	char *bufpos = buf;
1405	unsigned long apid;
1406	unsigned long apqi;
1407	unsigned long apid1;
1408	unsigned long apqi1;
1409	unsigned long napm_bits = matrix->apm_max + 1;
1410	unsigned long naqm_bits = matrix->aqm_max + 1;
1411	int nchars = 0;
1412	int n;
1413
1414	apid1 = find_first_bit_inv(matrix->apm, napm_bits);
1415	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);
1416
1417	if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) {
1418		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
1419			for_each_set_bit_inv(apqi, matrix->aqm,
1420					     naqm_bits) {
1421				n = sprintf(bufpos, "%02lx.%04lx\n", apid,
1422					    apqi);
1423				bufpos += n;
1424				nchars += n;
1425			}
1426		}
1427	} else if (apid1 < napm_bits) {
1428		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
1429			n = sprintf(bufpos, "%02lx.\n", apid);
1430			bufpos += n;
1431			nchars += n;
1432		}
1433	} else if (apqi1 < naqm_bits) {
1434		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) {
1435			n = sprintf(bufpos, ".%04lx\n", apqi);
1436			bufpos += n;
1437			nchars += n;
1438		}
1439	}
1440
1441	return nchars;
1442}
1443
1444static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
1445			   char *buf)
1446{
1447	ssize_t nchars;
1448	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1449
1450	mutex_lock(&matrix_dev->mdevs_lock);
1451	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);
1452	mutex_unlock(&matrix_dev->mdevs_lock);
1453
1454	return nchars;
1455}
1456static DEVICE_ATTR_RO(matrix);
1457
1458static ssize_t guest_matrix_show(struct device *dev,
1459				 struct device_attribute *attr, char *buf)
1460{
1461	ssize_t nchars;
1462	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1463
1464	mutex_lock(&matrix_dev->mdevs_lock);
1465	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf);
1466	mutex_unlock(&matrix_dev->mdevs_lock);
1467
1468	return nchars;
1469}
1470static DEVICE_ATTR_RO(guest_matrix);
1471
1472static struct attribute *vfio_ap_mdev_attrs[] = {
1473	&dev_attr_assign_adapter.attr,
1474	&dev_attr_unassign_adapter.attr,
1475	&dev_attr_assign_domain.attr,
1476	&dev_attr_unassign_domain.attr,
1477	&dev_attr_assign_control_domain.attr,
1478	&dev_attr_unassign_control_domain.attr,
1479	&dev_attr_control_domains.attr,
1480	&dev_attr_matrix.attr,
1481	&dev_attr_guest_matrix.attr,
1482	NULL,
1483};
1484
1485static struct attribute_group vfio_ap_mdev_attr_group = {
1486	.attrs = vfio_ap_mdev_attrs
1487};
1488
1489static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
1490	&vfio_ap_mdev_attr_group,
1491	NULL
1492};
1493
1494/**
1495 * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed
1496 * to manage AP resources for the guest whose state is represented by @kvm
1497 *
1498 * @matrix_mdev: a mediated matrix device
1499 * @kvm: reference to KVM instance
1500 *
1501 * Return: 0 if no other mediated matrix device has a reference to @kvm;
1502 * otherwise, returns an -EPERM.
1503 */
1504static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
1505				struct kvm *kvm)
1506{
1507	struct ap_matrix_mdev *m;
1508
1509	if (kvm->arch.crypto.crycbd) {
1510		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1511		kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
1512		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1513
1514		get_update_locks_for_kvm(kvm);
1515
1516		list_for_each_entry(m, &matrix_dev->mdev_list, node) {
1517			if (m != matrix_mdev && m->kvm == kvm) {
1518				release_update_locks_for_kvm(kvm);
1519				return -EPERM;
1520			}
1521		}
1522
1523		kvm_get_kvm(kvm);
1524		matrix_mdev->kvm = kvm;
1525		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1526
1527		release_update_locks_for_kvm(kvm);
1528	}
1529
1530	return 0;
1531}
1532
1533static void unmap_iova(struct ap_matrix_mdev *matrix_mdev, u64 iova, u64 length)
1534{
1535	struct ap_queue_table *qtable = &matrix_mdev->qtable;
1536	struct vfio_ap_queue *q;
1537	int loop_cursor;
1538
1539	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1540		if (q->saved_iova >= iova && q->saved_iova < iova + length)
1541			vfio_ap_irq_disable(q);
1542	}
1543}
1544
1545static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova,
1546				   u64 length)
1547{
1548	struct ap_matrix_mdev *matrix_mdev =
1549		container_of(vdev, struct ap_matrix_mdev, vdev);
1550
1551	mutex_lock(&matrix_dev->mdevs_lock);
1552
1553	unmap_iova(matrix_mdev, iova, length);
1554
1555	mutex_unlock(&matrix_dev->mdevs_lock);
1556}
1557
1558/**
1559 * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed
1560 * by @matrix_mdev.
1561 *
1562 * @matrix_mdev: a matrix mediated device
1563 */
1564static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
1565{
1566	struct kvm *kvm = matrix_mdev->kvm;
1567
1568	if (kvm && kvm->arch.crypto.crycbd) {
1569		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1570		kvm->arch.crypto.pqap_hook = NULL;
1571		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1572
1573		get_update_locks_for_kvm(kvm);
1574
1575		kvm_arch_crypto_clear_masks(kvm);
1576		vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
1577		kvm_put_kvm(kvm);
1578		matrix_mdev->kvm = NULL;
1579
1580		release_update_locks_for_kvm(kvm);
1581	}
1582}
1583
1584static struct vfio_ap_queue *vfio_ap_find_queue(int apqn)
1585{
1586	struct ap_queue *queue;
1587	struct vfio_ap_queue *q = NULL;
1588
1589	queue = ap_get_qdev(apqn);
1590	if (!queue)
1591		return NULL;
1592
1593	if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver)
1594		q = dev_get_drvdata(&queue->ap_dev.device);
1595
1596	put_device(&queue->ap_dev.device);
1597
1598	return q;
1599}
1600
1601static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q,
1602				    unsigned int retry)
1603{
1604	struct ap_queue_status status;
1605	int ret;
1606	int retry2 = 2;
1607
1608	if (!q)
1609		return 0;
1610retry_zapq:
1611	status = ap_zapq(q->apqn);
1612	q->reset_rc = status.response_code;
1613	switch (status.response_code) {
1614	case AP_RESPONSE_NORMAL:
1615		ret = 0;
1616		break;
1617	case AP_RESPONSE_RESET_IN_PROGRESS:
1618		if (retry--) {
1619			msleep(20);
1620			goto retry_zapq;
1621		}
1622		ret = -EBUSY;
1623		break;
1624	case AP_RESPONSE_Q_NOT_AVAIL:
1625	case AP_RESPONSE_DECONFIGURED:
1626	case AP_RESPONSE_CHECKSTOPPED:
1627		WARN_ONCE(status.irq_enabled,
1628			  "PQAP/ZAPQ for %02x.%04x failed with rc=%u while IRQ enabled",
1629			  AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
1630			  status.response_code);
1631		ret = -EBUSY;
1632		goto free_resources;
1633	default:
1634		/* things are really broken, give up */
1635		WARN(true,
1636		     "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n",
1637		     AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
1638		     status.response_code);
1639		return -EIO;
1640	}
1641
1642	/* wait for the reset to take effect */
1643	while (retry2--) {
1644		if (status.queue_empty && !status.irq_enabled)
1645			break;
1646		msleep(20);
1647		status = ap_tapq(q->apqn, NULL);
1648	}
1649	WARN_ONCE(retry2 <= 0, "unable to verify reset of queue %02x.%04x",
1650		  AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn));
1651
1652free_resources:
1653	vfio_ap_free_aqic_resources(q);
1654
1655	return ret;
1656}
1657
1658static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable)
1659{
1660	int ret, loop_cursor, rc = 0;
1661	struct vfio_ap_queue *q;
1662
1663	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1664		ret = vfio_ap_mdev_reset_queue(q, 1);
1665		/*
1666		 * Regardless whether a queue turns out to be busy, or
1667		 * is not operational, we need to continue resetting
1668		 * the remaining queues.
1669		 */
1670		if (ret)
1671			rc = ret;
1672	}
1673
1674	return rc;
1675}
1676
1677static int vfio_ap_mdev_open_device(struct vfio_device *vdev)
1678{
1679	struct ap_matrix_mdev *matrix_mdev =
1680		container_of(vdev, struct ap_matrix_mdev, vdev);
1681
1682	if (!vdev->kvm)
1683		return -EINVAL;
1684
1685	return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm);
1686}
1687
1688static void vfio_ap_mdev_close_device(struct vfio_device *vdev)
1689{
1690	struct ap_matrix_mdev *matrix_mdev =
1691		container_of(vdev, struct ap_matrix_mdev, vdev);
1692
1693	vfio_ap_mdev_unset_kvm(matrix_mdev);
1694}
1695
1696static int vfio_ap_mdev_get_device_info(unsigned long arg)
1697{
1698	unsigned long minsz;
1699	struct vfio_device_info info;
1700
1701	minsz = offsetofend(struct vfio_device_info, num_irqs);
1702
1703	if (copy_from_user(&info, (void __user *)arg, minsz))
1704		return -EFAULT;
1705
1706	if (info.argsz < minsz)
1707		return -EINVAL;
1708
1709	info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
1710	info.num_regions = 0;
1711	info.num_irqs = 0;
1712
1713	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
1714}
1715
1716static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
1717				    unsigned int cmd, unsigned long arg)
1718{
1719	struct ap_matrix_mdev *matrix_mdev =
1720		container_of(vdev, struct ap_matrix_mdev, vdev);
1721	int ret;
1722
1723	mutex_lock(&matrix_dev->mdevs_lock);
1724	switch (cmd) {
1725	case VFIO_DEVICE_GET_INFO:
1726		ret = vfio_ap_mdev_get_device_info(arg);
1727		break;
1728	case VFIO_DEVICE_RESET:
1729		ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
1730		break;
1731	default:
1732		ret = -EOPNOTSUPP;
1733		break;
1734	}
1735	mutex_unlock(&matrix_dev->mdevs_lock);
1736
1737	return ret;
1738}
1739
1740static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q)
1741{
1742	struct ap_matrix_mdev *matrix_mdev;
1743	unsigned long apid = AP_QID_CARD(q->apqn);
1744	unsigned long apqi = AP_QID_QUEUE(q->apqn);
1745
1746	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
1747		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&
1748		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))
1749			return matrix_mdev;
1750	}
1751
1752	return NULL;
1753}
1754
1755static ssize_t status_show(struct device *dev,
1756			   struct device_attribute *attr,
1757			   char *buf)
1758{
1759	ssize_t nchars = 0;
1760	struct vfio_ap_queue *q;
1761	struct ap_matrix_mdev *matrix_mdev;
1762	struct ap_device *apdev = to_ap_dev(dev);
1763
1764	mutex_lock(&matrix_dev->mdevs_lock);
1765	q = dev_get_drvdata(&apdev->device);
1766	matrix_mdev = vfio_ap_mdev_for_queue(q);
1767
1768	if (matrix_mdev) {
1769		if (matrix_mdev->kvm)
1770			nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1771					   AP_QUEUE_IN_USE);
1772		else
1773			nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1774					   AP_QUEUE_ASSIGNED);
1775	} else {
1776		nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1777				   AP_QUEUE_UNASSIGNED);
1778	}
1779
1780	mutex_unlock(&matrix_dev->mdevs_lock);
1781
1782	return nchars;
1783}
1784
1785static DEVICE_ATTR_RO(status);
1786
1787static struct attribute *vfio_queue_attrs[] = {
1788	&dev_attr_status.attr,
1789	NULL,
1790};
1791
1792static const struct attribute_group vfio_queue_attr_group = {
1793	.attrs = vfio_queue_attrs,
1794};
1795
1796static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
1797	.init = vfio_ap_mdev_init_dev,
1798	.open_device = vfio_ap_mdev_open_device,
1799	.close_device = vfio_ap_mdev_close_device,
1800	.ioctl = vfio_ap_mdev_ioctl,
1801	.dma_unmap = vfio_ap_mdev_dma_unmap,
1802	.bind_iommufd = vfio_iommufd_emulated_bind,
1803	.unbind_iommufd = vfio_iommufd_emulated_unbind,
1804	.attach_ioas = vfio_iommufd_emulated_attach_ioas,
1805};
1806
1807static struct mdev_driver vfio_ap_matrix_driver = {
1808	.device_api = VFIO_DEVICE_API_AP_STRING,
1809	.max_instances = MAX_ZDEV_ENTRIES_EXT,
1810	.driver = {
1811		.name = "vfio_ap_mdev",
1812		.owner = THIS_MODULE,
1813		.mod_name = KBUILD_MODNAME,
1814		.dev_groups = vfio_ap_mdev_attr_groups,
1815	},
1816	.probe = vfio_ap_mdev_probe,
1817	.remove = vfio_ap_mdev_remove,
1818};
1819
1820int vfio_ap_mdev_register(void)
1821{
1822	int ret;
1823
1824	ret = mdev_register_driver(&vfio_ap_matrix_driver);
1825	if (ret)
1826		return ret;
1827
1828	matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT;
1829	matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT;
1830	matrix_dev->mdev_types[0] = &matrix_dev->mdev_type;
1831	ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device,
1832				   &vfio_ap_matrix_driver,
1833				   matrix_dev->mdev_types, 1);
1834	if (ret)
1835		goto err_driver;
1836	return 0;
1837
1838err_driver:
1839	mdev_unregister_driver(&vfio_ap_matrix_driver);
1840	return ret;
1841}
1842
1843void vfio_ap_mdev_unregister(void)
1844{
1845	mdev_unregister_parent(&matrix_dev->parent);
1846	mdev_unregister_driver(&vfio_ap_matrix_driver);
1847}
1848
1849int vfio_ap_mdev_probe_queue(struct ap_device *apdev)
1850{
1851	int ret;
1852	struct vfio_ap_queue *q;
1853	struct ap_matrix_mdev *matrix_mdev;
1854
1855	ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group);
1856	if (ret)
1857		return ret;
1858
1859	q = kzalloc(sizeof(*q), GFP_KERNEL);
1860	if (!q)
1861		return -ENOMEM;
1862
1863	q->apqn = to_ap_queue(&apdev->device)->qid;
1864	q->saved_isc = VFIO_AP_ISC_INVALID;
1865	matrix_mdev = get_update_locks_by_apqn(q->apqn);
1866
1867	if (matrix_mdev) {
1868		vfio_ap_mdev_link_queue(matrix_mdev, q);
1869
1870		if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm,
1871					       matrix_mdev->matrix.aqm,
1872					       matrix_mdev))
1873			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1874	}
1875	dev_set_drvdata(&apdev->device, q);
1876	release_update_locks_for_mdev(matrix_mdev);
1877
1878	return 0;
1879}
1880
1881void vfio_ap_mdev_remove_queue(struct ap_device *apdev)
1882{
1883	unsigned long apid, apqi;
1884	struct vfio_ap_queue *q;
1885	struct ap_matrix_mdev *matrix_mdev;
1886
1887	sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
1888	q = dev_get_drvdata(&apdev->device);
1889	get_update_locks_for_queue(q);
1890	matrix_mdev = q->matrix_mdev;
1891
1892	if (matrix_mdev) {
1893		vfio_ap_unlink_queue_fr_mdev(q);
1894
1895		apid = AP_QID_CARD(q->apqn);
1896		apqi = AP_QID_QUEUE(q->apqn);
1897
1898		/*
1899		 * If the queue is assigned to the guest's APCB, then remove
1900		 * the adapter's APID from the APCB and hot it into the guest.
1901		 */
1902		if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1903		    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
1904			clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
1905			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1906		}
1907	}
1908
1909	vfio_ap_mdev_reset_queue(q, 1);
1910	dev_set_drvdata(&apdev->device, NULL);
1911	kfree(q);
1912	release_update_locks_for_mdev(matrix_mdev);
1913}
1914
1915/**
1916 * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is
1917 *				 assigned to a mediated device under the control
1918 *				 of the vfio_ap device driver.
1919 *
1920 * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check.
1921 * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check.
1922 *
1923 * Return:
1924 *	* -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are
1925 *	  assigned to a mediated device under the control of the vfio_ap
1926 *	  device driver.
1927 *	* Otherwise, return 0.
1928 */
1929int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm)
1930{
1931	int ret;
1932
1933	mutex_lock(&matrix_dev->guests_lock);
1934	mutex_lock(&matrix_dev->mdevs_lock);
1935	ret = vfio_ap_mdev_verify_no_sharing(apm, aqm);
1936	mutex_unlock(&matrix_dev->mdevs_lock);
1937	mutex_unlock(&matrix_dev->guests_lock);
1938
1939	return ret;
1940}
1941
1942/**
1943 * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control
1944 *				 domains that have been removed from the host's
1945 *				 AP configuration from a guest.
1946 *
1947 * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest.
1948 * @aprem: the adapters that have been removed from the host's AP configuration
1949 * @aqrem: the domains that have been removed from the host's AP configuration
1950 * @cdrem: the control domains that have been removed from the host's AP
1951 *	   configuration.
1952 */
1953static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev,
1954					unsigned long *aprem,
1955					unsigned long *aqrem,
1956					unsigned long *cdrem)
1957{
1958	int do_hotplug = 0;
1959
1960	if (!bitmap_empty(aprem, AP_DEVICES)) {
1961		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm,
1962					    matrix_mdev->shadow_apcb.apm,
1963					    aprem, AP_DEVICES);
1964	}
1965
1966	if (!bitmap_empty(aqrem, AP_DOMAINS)) {
1967		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm,
1968					    matrix_mdev->shadow_apcb.aqm,
1969					    aqrem, AP_DEVICES);
1970	}
1971
1972	if (!bitmap_empty(cdrem, AP_DOMAINS))
1973		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm,
1974					    matrix_mdev->shadow_apcb.adm,
1975					    cdrem, AP_DOMAINS);
1976
1977	if (do_hotplug)
1978		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1979}
1980
1981/**
1982 * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters,
1983 *			     domains and control domains that have been removed
1984 *			     from the host AP configuration and unplugs them
1985 *			     from those guests.
1986 *
1987 * @ap_remove:	bitmap specifying which adapters have been removed from the host
1988 *		config.
1989 * @aq_remove:	bitmap specifying which domains have been removed from the host
1990 *		config.
1991 * @cd_remove:	bitmap specifying which control domains have been removed from
1992 *		the host config.
1993 */
1994static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove,
1995				    unsigned long *aq_remove,
1996				    unsigned long *cd_remove)
1997{
1998	struct ap_matrix_mdev *matrix_mdev;
1999	DECLARE_BITMAP(aprem, AP_DEVICES);
2000	DECLARE_BITMAP(aqrem, AP_DOMAINS);
2001	DECLARE_BITMAP(cdrem, AP_DOMAINS);
2002	int do_remove = 0;
2003
2004	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2005		mutex_lock(&matrix_mdev->kvm->lock);
2006		mutex_lock(&matrix_dev->mdevs_lock);
2007
2008		do_remove |= bitmap_and(aprem, ap_remove,
2009					  matrix_mdev->matrix.apm,
2010					  AP_DEVICES);
2011		do_remove |= bitmap_and(aqrem, aq_remove,
2012					  matrix_mdev->matrix.aqm,
2013					  AP_DOMAINS);
2014		do_remove |= bitmap_andnot(cdrem, cd_remove,
2015					     matrix_mdev->matrix.adm,
2016					     AP_DOMAINS);
2017
2018		if (do_remove)
2019			vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem,
2020						    cdrem);
2021
2022		mutex_unlock(&matrix_dev->mdevs_lock);
2023		mutex_unlock(&matrix_mdev->kvm->lock);
2024	}
2025}
2026
2027/**
2028 * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and
2029 *				control domains from the host AP configuration
2030 *				by unplugging them from the guests that are
2031 *				using them.
2032 * @cur_config_info: the current host AP configuration information
2033 * @prev_config_info: the previous host AP configuration information
2034 */
2035static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info,
2036				       struct ap_config_info *prev_config_info)
2037{
2038	int do_remove;
2039	DECLARE_BITMAP(aprem, AP_DEVICES);
2040	DECLARE_BITMAP(aqrem, AP_DOMAINS);
2041	DECLARE_BITMAP(cdrem, AP_DOMAINS);
2042
2043	do_remove = bitmap_andnot(aprem,
2044				  (unsigned long *)prev_config_info->apm,
2045				  (unsigned long *)cur_config_info->apm,
2046				  AP_DEVICES);
2047	do_remove |= bitmap_andnot(aqrem,
2048				   (unsigned long *)prev_config_info->aqm,
2049				   (unsigned long *)cur_config_info->aqm,
2050				   AP_DEVICES);
2051	do_remove |= bitmap_andnot(cdrem,
2052				   (unsigned long *)prev_config_info->adm,
2053				   (unsigned long *)cur_config_info->adm,
2054				   AP_DEVICES);
2055
2056	if (do_remove)
2057		vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem);
2058}
2059
2060/**
2061 * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that
2062 *				 are older than AP type 10 (CEX4).
2063 * @apm: a bitmap of the APIDs to examine
2064 * @aqm: a bitmap of the APQIs of the queues to query for the AP type.
2065 */
2066static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm)
2067{
2068	bool apid_cleared;
2069	struct ap_queue_status status;
2070	unsigned long apid, apqi, info;
2071	int qtype, qtype_mask = 0xff000000;
2072
2073	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
2074		apid_cleared = false;
2075
2076		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
2077			status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info);
2078			switch (status.response_code) {
2079			/*
2080			 * According to the architecture in each case
2081			 * below, the queue's info should be filled.
2082			 */
2083			case AP_RESPONSE_NORMAL:
2084			case AP_RESPONSE_RESET_IN_PROGRESS:
2085			case AP_RESPONSE_DECONFIGURED:
2086			case AP_RESPONSE_CHECKSTOPPED:
2087			case AP_RESPONSE_BUSY:
2088				qtype = info & qtype_mask;
2089
2090				/*
2091				 * The vfio_ap device driver only
2092				 * supports CEX4 and newer adapters, so
2093				 * remove the APID if the adapter is
2094				 * older than a CEX4.
2095				 */
2096				if (qtype < AP_DEVICE_TYPE_CEX4) {
2097					clear_bit_inv(apid, apm);
2098					apid_cleared = true;
2099				}
2100
2101				break;
2102
2103			default:
2104				/*
2105				 * If we don't know the adapter type,
2106				 * clear its APID since it can't be
2107				 * determined whether the vfio_ap
2108				 * device driver supports it.
2109				 */
2110				clear_bit_inv(apid, apm);
2111				apid_cleared = true;
2112				break;
2113			}
2114
2115			/*
2116			 * If we've already cleared the APID from the apm, there
2117			 * is no need to continue examining the remainin AP
2118			 * queues to determine the type of the adapter.
2119			 */
2120			if (apid_cleared)
2121				continue;
2122		}
2123	}
2124}
2125
2126/**
2127 * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and
2128 *			  control domains that have been added to the host's
2129 *			  AP configuration for each matrix mdev to which they
2130 *			  are assigned.
2131 *
2132 * @apm_add: a bitmap specifying the adapters that have been added to the AP
2133 *	     configuration.
2134 * @aqm_add: a bitmap specifying the domains that have been added to the AP
2135 *	     configuration.
2136 * @adm_add: a bitmap specifying the control domains that have been added to the
2137 *	     AP configuration.
2138 */
2139static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add,
2140				 unsigned long *adm_add)
2141{
2142	struct ap_matrix_mdev *matrix_mdev;
2143
2144	if (list_empty(&matrix_dev->mdev_list))
2145		return;
2146
2147	vfio_ap_filter_apid_by_qtype(apm_add, aqm_add);
2148
2149	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2150		bitmap_and(matrix_mdev->apm_add,
2151			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);
2152		bitmap_and(matrix_mdev->aqm_add,
2153			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);
2154		bitmap_and(matrix_mdev->adm_add,
2155			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);
2156	}
2157}
2158
2159/**
2160 * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and
2161 *			     control domains to the host AP configuration
2162 *			     by updating the bitmaps that specify what adapters,
2163 *			     domains and control domains have been added so they
2164 *			     can be hot plugged into the guest when the AP bus
2165 *			     scan completes (see vfio_ap_on_scan_complete
2166 *			     function).
2167 * @cur_config_info: the current AP configuration information
2168 * @prev_config_info: the previous AP configuration information
2169 */
2170static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info,
2171				    struct ap_config_info *prev_config_info)
2172{
2173	bool do_add;
2174	DECLARE_BITMAP(apm_add, AP_DEVICES);
2175	DECLARE_BITMAP(aqm_add, AP_DOMAINS);
2176	DECLARE_BITMAP(adm_add, AP_DOMAINS);
2177
2178	do_add = bitmap_andnot(apm_add,
2179			       (unsigned long *)cur_config_info->apm,
2180			       (unsigned long *)prev_config_info->apm,
2181			       AP_DEVICES);
2182	do_add |= bitmap_andnot(aqm_add,
2183				(unsigned long *)cur_config_info->aqm,
2184				(unsigned long *)prev_config_info->aqm,
2185				AP_DOMAINS);
2186	do_add |= bitmap_andnot(adm_add,
2187				(unsigned long *)cur_config_info->adm,
2188				(unsigned long *)prev_config_info->adm,
2189				AP_DOMAINS);
2190
2191	if (do_add)
2192		vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add);
2193}
2194
2195/**
2196 * vfio_ap_on_cfg_changed - handles notification of changes to the host AP
2197 *			    configuration.
2198 *
2199 * @cur_cfg_info: the current host AP configuration
2200 * @prev_cfg_info: the previous host AP configuration
2201 */
2202void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info,
2203			    struct ap_config_info *prev_cfg_info)
2204{
2205	if (!cur_cfg_info || !prev_cfg_info)
2206		return;
2207
2208	mutex_lock(&matrix_dev->guests_lock);
2209
2210	vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info);
2211	vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info);
2212	memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info));
2213
2214	mutex_unlock(&matrix_dev->guests_lock);
2215}
2216
2217static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev)
2218{
2219	bool do_hotplug = false;
2220	int filter_domains = 0;
2221	int filter_adapters = 0;
2222	DECLARE_BITMAP(apm, AP_DEVICES);
2223	DECLARE_BITMAP(aqm, AP_DOMAINS);
2224
2225	mutex_lock(&matrix_mdev->kvm->lock);
2226	mutex_lock(&matrix_dev->mdevs_lock);
2227
2228	filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm,
2229				     matrix_mdev->apm_add, AP_DEVICES);
2230	filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm,
2231				    matrix_mdev->aqm_add, AP_DOMAINS);
2232
2233	if (filter_adapters && filter_domains)
2234		do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev);
2235	else if (filter_adapters)
2236		do_hotplug |=
2237			vfio_ap_mdev_filter_matrix(apm,
2238						   matrix_mdev->shadow_apcb.aqm,
2239						   matrix_mdev);
2240	else
2241		do_hotplug |=
2242			vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm,
2243						   aqm, matrix_mdev);
2244
2245	if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add,
2246			      AP_DOMAINS))
2247		do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev);
2248
2249	if (do_hotplug)
2250		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2251
2252	mutex_unlock(&matrix_dev->mdevs_lock);
2253	mutex_unlock(&matrix_mdev->kvm->lock);
2254}
2255
2256void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,
2257			      struct ap_config_info *old_config_info)
2258{
2259	struct ap_matrix_mdev *matrix_mdev;
2260
2261	mutex_lock(&matrix_dev->guests_lock);
2262
2263	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2264		if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) &&
2265		    bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) &&
2266		    bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS))
2267			continue;
2268
2269		vfio_ap_mdev_hot_plug_cfg(matrix_mdev);
2270		bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES);
2271		bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS);
2272		bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS);
2273	}
2274
2275	mutex_unlock(&matrix_dev->guests_lock);
2276}