1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */
   3
   4#include <linux/init.h>
   5#include <linux/kernel.h>
   6#include <linux/module.h>
   7#include <linux/pci.h>
   8#include <linux/device.h>
   9#include <linux/iommu.h>
  10#include <uapi/linux/idxd.h>
  11#include <linux/highmem.h>
  12#include <linux/sched/smt.h>
  13#include <crypto/internal/acompress.h>
  14
  15#include "idxd.h"
  16#include "iaa_crypto.h"
  17#include "iaa_crypto_stats.h"
  18
  19#ifdef pr_fmt
  20#undef pr_fmt
  21#endif
  22
  23#define pr_fmt(fmt)			"idxd: " IDXD_SUBDRIVER_NAME ": " fmt
  24
  25#define IAA_ALG_PRIORITY               300
  26
  27/* number of iaa instances probed */
  28static unsigned int nr_iaa;
  29static unsigned int nr_cpus;
  30static unsigned int nr_nodes;
  31static unsigned int nr_cpus_per_node;
  32
  33/* Number of physical cpus sharing each iaa instance */
  34static unsigned int cpus_per_iaa;
  35
  36static struct crypto_comp *deflate_generic_tfm;
  37
  38/* Per-cpu lookup table for balanced wqs */
  39static struct wq_table_entry __percpu *wq_table;
  40
  41static struct idxd_wq *wq_table_next_wq(int cpu)
  42{
  43	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
  44
  45	if (++entry->cur_wq >= entry->n_wqs)
  46		entry->cur_wq = 0;
  47
  48	if (!entry->wqs[entry->cur_wq])
  49		return NULL;
  50
  51	pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__,
  52		 entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id,
  53		 entry->wqs[entry->cur_wq]->id, cpu);
  54
  55	return entry->wqs[entry->cur_wq];
  56}
  57
  58static void wq_table_add(int cpu, struct idxd_wq *wq)
  59{
  60	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
  61
  62	if (WARN_ON(entry->n_wqs == entry->max_wqs))
  63		return;
  64
  65	entry->wqs[entry->n_wqs++] = wq;
  66
  67	pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__,
  68		 entry->wqs[entry->n_wqs - 1]->idxd->id,
  69		 entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu);
  70}
  71
  72static void wq_table_free_entry(int cpu)
  73{
  74	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
  75
  76	kfree(entry->wqs);
  77	memset(entry, 0, sizeof(*entry));
  78}
  79
  80static void wq_table_clear_entry(int cpu)
  81{
  82	struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
  83
  84	entry->n_wqs = 0;
  85	entry->cur_wq = 0;
  86	memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *));
  87}
  88
  89LIST_HEAD(iaa_devices);
  90DEFINE_MUTEX(iaa_devices_lock);
  91
  92/* If enabled, IAA hw crypto algos are registered, unavailable otherwise */
  93static bool iaa_crypto_enabled;
  94static bool iaa_crypto_registered;
  95
  96/* Verify results of IAA compress or not */
  97static bool iaa_verify_compress = true;
  98
  99static ssize_t verify_compress_show(struct device_driver *driver, char *buf)
 100{
 101	return sprintf(buf, "%d\n", iaa_verify_compress);
 102}
 103
 104static ssize_t verify_compress_store(struct device_driver *driver,
 105				     const char *buf, size_t count)
 106{
 107	int ret = -EBUSY;
 108
 109	mutex_lock(&iaa_devices_lock);
 110
 111	if (iaa_crypto_enabled)
 112		goto out;
 113
 114	ret = kstrtobool(buf, &iaa_verify_compress);
 115	if (ret)
 116		goto out;
 117
 118	ret = count;
 119out:
 120	mutex_unlock(&iaa_devices_lock);
 121
 122	return ret;
 123}
 124static DRIVER_ATTR_RW(verify_compress);
 125
 126/*
 127 * The iaa crypto driver supports three 'sync' methods determining how
 128 * compressions and decompressions are performed:
 129 *
 130 * - sync:      the compression or decompression completes before
 131 *              returning.  This is the mode used by the async crypto
 132 *              interface when the sync mode is set to 'sync' and by
 133 *              the sync crypto interface regardless of setting.
 134 *
 135 * - async:     the compression or decompression is submitted and returns
 136 *              immediately.  Completion interrupts are not used so
 137 *              the caller is responsible for polling the descriptor
 138 *              for completion.  This mode is applicable to only the
 139 *              async crypto interface and is ignored for anything
 140 *              else.
 141 *
 142 * - async_irq: the compression or decompression is submitted and
 143 *              returns immediately.  Completion interrupts are
 144 *              enabled so the caller can wait for the completion and
 145 *              yield to other threads.  When the compression or
 146 *              decompression completes, the completion is signaled
 147 *              and the caller awakened.  This mode is applicable to
 148 *              only the async crypto interface and is ignored for
 149 *              anything else.
 150 *
 151 * These modes can be set using the iaa_crypto sync_mode driver
 152 * attribute.
 153 */
 154
 155/* Use async mode */
 156static bool async_mode;
 157/* Use interrupts */
 158static bool use_irq;
 159
 160/**
 161 * set_iaa_sync_mode - Set IAA sync mode
 162 * @name: The name of the sync mode
 163 *
 164 * Make the IAA sync mode named @name the current sync mode used by
 165 * compression/decompression.
 166 */
 167
 168static int set_iaa_sync_mode(const char *name)
 169{
 170	int ret = 0;
 171
 172	if (sysfs_streq(name, "sync")) {
 173		async_mode = false;
 174		use_irq = false;
 175	} else if (sysfs_streq(name, "async")) {
 176		async_mode = true;
 177		use_irq = false;
 178	} else if (sysfs_streq(name, "async_irq")) {
 179		async_mode = true;
 180		use_irq = true;
 181	} else {
 182		ret = -EINVAL;
 183	}
 184
 185	return ret;
 186}
 187
 188static ssize_t sync_mode_show(struct device_driver *driver, char *buf)
 189{
 190	int ret = 0;
 191
 192	if (!async_mode && !use_irq)
 193		ret = sprintf(buf, "%s\n", "sync");
 194	else if (async_mode && !use_irq)
 195		ret = sprintf(buf, "%s\n", "async");
 196	else if (async_mode && use_irq)
 197		ret = sprintf(buf, "%s\n", "async_irq");
 198
 199	return ret;
 200}
 201
 202static ssize_t sync_mode_store(struct device_driver *driver,
 203			       const char *buf, size_t count)
 204{
 205	int ret = -EBUSY;
 206
 207	mutex_lock(&iaa_devices_lock);
 208
 209	if (iaa_crypto_enabled)
 210		goto out;
 211
 212	ret = set_iaa_sync_mode(buf);
 213	if (ret == 0)
 214		ret = count;
 215out:
 216	mutex_unlock(&iaa_devices_lock);
 217
 218	return ret;
 219}
 220static DRIVER_ATTR_RW(sync_mode);
 221
 222static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX];
 223
 224static int find_empty_iaa_compression_mode(void)
 225{
 226	int i = -EINVAL;
 227
 228	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
 229		if (iaa_compression_modes[i])
 230			continue;
 231		break;
 232	}
 233
 234	return i;
 235}
 236
 237static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx)
 238{
 239	struct iaa_compression_mode *mode;
 240	int i;
 241
 242	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
 243		mode = iaa_compression_modes[i];
 244		if (!mode)
 245			continue;
 246
 247		if (!strcmp(mode->name, name)) {
 248			*idx = i;
 249			return iaa_compression_modes[i];
 250		}
 251	}
 252
 253	return NULL;
 254}
 255
 256static void free_iaa_compression_mode(struct iaa_compression_mode *mode)
 257{
 258	kfree(mode->name);
 259	kfree(mode->ll_table);
 260	kfree(mode->d_table);
 261	kfree(mode->header_table);
 262
 263	kfree(mode);
 264}
 265
 266/*
 267 * IAA Compression modes are defined by an ll_table, a d_table, and an
 268 * optional header_table.  These tables are typically generated and
 269 * captured using statistics collected from running actual
 270 * compress/decompress workloads.
 271 *
 272 * A module or other kernel code can add and remove compression modes
 273 * with a given name using the exported @add_iaa_compression_mode()
 274 * and @remove_iaa_compression_mode functions.
 275 *
 276 * When a new compression mode is added, the tables are saved in a
 277 * global compression mode list.  When IAA devices are added, a
 278 * per-IAA device dma mapping is created for each IAA device, for each
 279 * compression mode.  These are the tables used to do the actual
 280 * compression/deccompression and are unmapped if/when the devices are
 281 * removed.  Currently, compression modes must be added before any
 282 * device is added, and removed after all devices have been removed.
 283 */
 284
 285/**
 286 * remove_iaa_compression_mode - Remove an IAA compression mode
 287 * @name: The name the compression mode will be known as
 288 *
 289 * Remove the IAA compression mode named @name.
 290 */
 291void remove_iaa_compression_mode(const char *name)
 292{
 293	struct iaa_compression_mode *mode;
 294	int idx;
 295
 296	mutex_lock(&iaa_devices_lock);
 297
 298	if (!list_empty(&iaa_devices))
 299		goto out;
 300
 301	mode = find_iaa_compression_mode(name, &idx);
 302	if (mode) {
 303		free_iaa_compression_mode(mode);
 304		iaa_compression_modes[idx] = NULL;
 305	}
 306out:
 307	mutex_unlock(&iaa_devices_lock);
 308}
 309EXPORT_SYMBOL_GPL(remove_iaa_compression_mode);
 310
 311/**
 312 * add_iaa_compression_mode - Add an IAA compression mode
 313 * @name: The name the compression mode will be known as
 314 * @ll_table: The ll table
 315 * @ll_table_size: The ll table size in bytes
 316 * @d_table: The d table
 317 * @d_table_size: The d table size in bytes
 318 * @header_table: Optional header table
 319 * @header_table_size: Optional header table size in bytes
 320 * @gen_decomp_table_flags: Otional flags used to generate the decomp table
 321 * @init: Optional callback function to init the compression mode data
 322 * @free: Optional callback function to free the compression mode data
 323 *
 324 * Add a new IAA compression mode named @name.
 325 *
 326 * Returns 0 if successful, errcode otherwise.
 327 */
 328int add_iaa_compression_mode(const char *name,
 329			     const u32 *ll_table,
 330			     int ll_table_size,
 331			     const u32 *d_table,
 332			     int d_table_size,
 333			     const u8 *header_table,
 334			     int header_table_size,
 335			     u16 gen_decomp_table_flags,
 336			     iaa_dev_comp_init_fn_t init,
 337			     iaa_dev_comp_free_fn_t free)
 338{
 339	struct iaa_compression_mode *mode;
 340	int idx, ret = -ENOMEM;
 341
 342	mutex_lock(&iaa_devices_lock);
 343
 344	if (!list_empty(&iaa_devices)) {
 345		ret = -EBUSY;
 346		goto out;
 347	}
 348
 349	mode = kzalloc(sizeof(*mode), GFP_KERNEL);
 350	if (!mode)
 351		goto out;
 352
 353	mode->name = kstrdup(name, GFP_KERNEL);
 354	if (!mode->name)
 355		goto free;
 356
 357	if (ll_table) {
 358		mode->ll_table = kzalloc(ll_table_size, GFP_KERNEL);
 359		if (!mode->ll_table)
 360			goto free;
 361		memcpy(mode->ll_table, ll_table, ll_table_size);
 362		mode->ll_table_size = ll_table_size;
 363	}
 364
 365	if (d_table) {
 366		mode->d_table = kzalloc(d_table_size, GFP_KERNEL);
 367		if (!mode->d_table)
 368			goto free;
 369		memcpy(mode->d_table, d_table, d_table_size);
 370		mode->d_table_size = d_table_size;
 371	}
 372
 373	if (header_table) {
 374		mode->header_table = kzalloc(header_table_size, GFP_KERNEL);
 375		if (!mode->header_table)
 376			goto free;
 377		memcpy(mode->header_table, header_table, header_table_size);
 378		mode->header_table_size = header_table_size;
 379	}
 380
 381	mode->gen_decomp_table_flags = gen_decomp_table_flags;
 382
 383	mode->init = init;
 384	mode->free = free;
 385
 386	idx = find_empty_iaa_compression_mode();
 387	if (idx < 0)
 388		goto free;
 389
 390	pr_debug("IAA compression mode %s added at idx %d\n",
 391		 mode->name, idx);
 392
 393	iaa_compression_modes[idx] = mode;
 394
 395	ret = 0;
 396out:
 397	mutex_unlock(&iaa_devices_lock);
 398
 399	return ret;
 400free:
 401	free_iaa_compression_mode(mode);
 402	goto out;
 403}
 404EXPORT_SYMBOL_GPL(add_iaa_compression_mode);
 405
 406static struct iaa_device_compression_mode *
 407get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx)
 408{
 409	return iaa_device->compression_modes[idx];
 410}
 411
 412static void free_device_compression_mode(struct iaa_device *iaa_device,
 413					 struct iaa_device_compression_mode *device_mode)
 414{
 415	size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
 416	struct device *dev = &iaa_device->idxd->pdev->dev;
 417
 418	kfree(device_mode->name);
 419
 420	if (device_mode->aecs_comp_table)
 421		dma_free_coherent(dev, size, device_mode->aecs_comp_table,
 422				  device_mode->aecs_comp_table_dma_addr);
 423	if (device_mode->aecs_decomp_table)
 424		dma_free_coherent(dev, size, device_mode->aecs_decomp_table,
 425				  device_mode->aecs_decomp_table_dma_addr);
 426
 427	kfree(device_mode);
 428}
 429
 430#define IDXD_OP_FLAG_AECS_RW_TGLS       0x400000
 431#define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC)
 432#define IAX_AECS_COMPRESS_FLAG	(IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
 433#define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
 434#define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \
 435						IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \
 436						IDXD_OP_FLAG_AECS_RW_TGLS)
 437
 438static int check_completion(struct device *dev,
 439			    struct iax_completion_record *comp,
 440			    bool compress,
 441			    bool only_once);
 442
 443static int decompress_header(struct iaa_device_compression_mode *device_mode,
 444			     struct iaa_compression_mode *mode,
 445			     struct idxd_wq *wq)
 446{
 447	dma_addr_t src_addr, src2_addr;
 448	struct idxd_desc *idxd_desc;
 449	struct iax_hw_desc *desc;
 450	struct device *dev;
 451	int ret = 0;
 452
 453	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
 454	if (IS_ERR(idxd_desc))
 455		return PTR_ERR(idxd_desc);
 456
 457	desc = idxd_desc->iax_hw;
 458
 459	dev = &wq->idxd->pdev->dev;
 460
 461	src_addr = dma_map_single(dev, (void *)mode->header_table,
 462				  mode->header_table_size, DMA_TO_DEVICE);
 463	dev_dbg(dev, "%s: mode->name %s, src_addr %llx, dev %p, src %p, slen %d\n",
 464		__func__, mode->name, src_addr,	dev,
 465		mode->header_table, mode->header_table_size);
 466	if (unlikely(dma_mapping_error(dev, src_addr))) {
 467		dev_dbg(dev, "dma_map_single err, exiting\n");
 468		ret = -ENOMEM;
 469		return ret;
 470	}
 471
 472	desc->flags = IAX_AECS_GEN_FLAG;
 473	desc->opcode = IAX_OPCODE_DECOMPRESS;
 474
 475	desc->src1_addr = (u64)src_addr;
 476	desc->src1_size = mode->header_table_size;
 477
 478	src2_addr = device_mode->aecs_decomp_table_dma_addr;
 479	desc->src2_addr = (u64)src2_addr;
 480	desc->src2_size = 1088;
 481	dev_dbg(dev, "%s: mode->name %s, src2_addr %llx, dev %p, src2_size %d\n",
 482		__func__, mode->name, desc->src2_addr, dev, desc->src2_size);
 483	desc->max_dst_size = 0; // suppressed output
 484
 485	desc->decompr_flags = mode->gen_decomp_table_flags;
 486
 487	desc->priv = 0;
 488
 489	desc->completion_addr = idxd_desc->compl_dma;
 490
 491	ret = idxd_submit_desc(wq, idxd_desc);
 492	if (ret) {
 493		pr_err("%s: submit_desc failed ret=0x%x\n", __func__, ret);
 494		goto out;
 495	}
 496
 497	ret = check_completion(dev, idxd_desc->iax_completion, false, false);
 498	if (ret)
 499		dev_dbg(dev, "%s: mode->name %s check_completion failed ret=%d\n",
 500			__func__, mode->name, ret);
 501	else
 502		dev_dbg(dev, "%s: mode->name %s succeeded\n", __func__,
 503			mode->name);
 504out:
 505	dma_unmap_single(dev, src_addr, 1088, DMA_TO_DEVICE);
 506
 507	return ret;
 508}
 509
 510static int init_device_compression_mode(struct iaa_device *iaa_device,
 511					struct iaa_compression_mode *mode,
 512					int idx, struct idxd_wq *wq)
 513{
 514	size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
 515	struct device *dev = &iaa_device->idxd->pdev->dev;
 516	struct iaa_device_compression_mode *device_mode;
 517	int ret = -ENOMEM;
 518
 519	device_mode = kzalloc(sizeof(*device_mode), GFP_KERNEL);
 520	if (!device_mode)
 521		return -ENOMEM;
 522
 523	device_mode->name = kstrdup(mode->name, GFP_KERNEL);
 524	if (!device_mode->name)
 525		goto free;
 526
 527	device_mode->aecs_comp_table = dma_alloc_coherent(dev, size,
 528							  &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL);
 529	if (!device_mode->aecs_comp_table)
 530		goto free;
 531
 532	device_mode->aecs_decomp_table = dma_alloc_coherent(dev, size,
 533							    &device_mode->aecs_decomp_table_dma_addr, GFP_KERNEL);
 534	if (!device_mode->aecs_decomp_table)
 535		goto free;
 536
 537	/* Add Huffman table to aecs */
 538	memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table));
 539	memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size);
 540	memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size);
 541
 542	if (mode->header_table) {
 543		ret = decompress_header(device_mode, mode, wq);
 544		if (ret) {
 545			pr_debug("iaa header decompression failed: ret=%d\n", ret);
 546			goto free;
 547		}
 548	}
 549
 550	if (mode->init) {
 551		ret = mode->init(device_mode);
 552		if (ret)
 553			goto free;
 554	}
 555
 556	/* mode index should match iaa_compression_modes idx */
 557	iaa_device->compression_modes[idx] = device_mode;
 558
 559	pr_debug("IAA %s compression mode initialized for iaa device %d\n",
 560		 mode->name, iaa_device->idxd->id);
 561
 562	ret = 0;
 563out:
 564	return ret;
 565free:
 566	pr_debug("IAA %s compression mode initialization failed for iaa device %d\n",
 567		 mode->name, iaa_device->idxd->id);
 568
 569	free_device_compression_mode(iaa_device, device_mode);
 570	goto out;
 571}
 572
 573static int init_device_compression_modes(struct iaa_device *iaa_device,
 574					 struct idxd_wq *wq)
 575{
 576	struct iaa_compression_mode *mode;
 577	int i, ret = 0;
 578
 579	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
 580		mode = iaa_compression_modes[i];
 581		if (!mode)
 582			continue;
 583
 584		ret = init_device_compression_mode(iaa_device, mode, i, wq);
 585		if (ret)
 586			break;
 587	}
 588
 589	return ret;
 590}
 591
 592static void remove_device_compression_modes(struct iaa_device *iaa_device)
 593{
 594	struct iaa_device_compression_mode *device_mode;
 595	int i;
 596
 597	for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
 598		device_mode = iaa_device->compression_modes[i];
 599		if (!device_mode)
 600			continue;
 601
 602		free_device_compression_mode(iaa_device, device_mode);
 603		iaa_device->compression_modes[i] = NULL;
 604		if (iaa_compression_modes[i]->free)
 605			iaa_compression_modes[i]->free(device_mode);
 606	}
 607}
 608
 609static struct iaa_device *iaa_device_alloc(void)
 610{
 611	struct iaa_device *iaa_device;
 612
 613	iaa_device = kzalloc(sizeof(*iaa_device), GFP_KERNEL);
 614	if (!iaa_device)
 615		return NULL;
 616
 617	INIT_LIST_HEAD(&iaa_device->wqs);
 618
 619	return iaa_device;
 620}
 621
 622static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
 623{
 624	struct iaa_wq *iaa_wq;
 625
 626	list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
 627		if (iaa_wq->wq == wq)
 628			return true;
 629	}
 630
 631	return false;
 632}
 633
 634static struct iaa_device *add_iaa_device(struct idxd_device *idxd)
 635{
 636	struct iaa_device *iaa_device;
 637
 638	iaa_device = iaa_device_alloc();
 639	if (!iaa_device)
 640		return NULL;
 641
 642	iaa_device->idxd = idxd;
 643
 644	list_add_tail(&iaa_device->list, &iaa_devices);
 645
 646	nr_iaa++;
 647
 648	return iaa_device;
 649}
 650
 651static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq)
 652{
 653	int ret = 0;
 654
 655	ret = init_device_compression_modes(iaa_device, iaa_wq->wq);
 656	if (ret)
 657		return ret;
 658
 659	return ret;
 660}
 661
 662static void del_iaa_device(struct iaa_device *iaa_device)
 663{
 664	list_del(&iaa_device->list);
 665
 666	nr_iaa--;
 667}
 668
 669static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq,
 670		      struct iaa_wq **new_wq)
 671{
 672	struct idxd_device *idxd = iaa_device->idxd;
 673	struct pci_dev *pdev = idxd->pdev;
 674	struct device *dev = &pdev->dev;
 675	struct iaa_wq *iaa_wq;
 676
 677	iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL);
 678	if (!iaa_wq)
 679		return -ENOMEM;
 680
 681	iaa_wq->wq = wq;
 682	iaa_wq->iaa_device = iaa_device;
 683	idxd_wq_set_private(wq, iaa_wq);
 684
 685	list_add_tail(&iaa_wq->list, &iaa_device->wqs);
 686
 687	iaa_device->n_wq++;
 688
 689	if (new_wq)
 690		*new_wq = iaa_wq;
 691
 692	dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n",
 693		wq->id, iaa_device->idxd->id, iaa_device->n_wq);
 694
 695	return 0;
 696}
 697
 698static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
 699{
 700	struct idxd_device *idxd = iaa_device->idxd;
 701	struct pci_dev *pdev = idxd->pdev;
 702	struct device *dev = &pdev->dev;
 703	struct iaa_wq *iaa_wq;
 704
 705	list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
 706		if (iaa_wq->wq == wq) {
 707			list_del(&iaa_wq->list);
 708			iaa_device->n_wq--;
 709
 710			dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n",
 711				wq->id, iaa_device->idxd->id,
 712				iaa_device->n_wq, nr_iaa);
 713
 714			if (iaa_device->n_wq == 0)
 715				del_iaa_device(iaa_device);
 716			break;
 717		}
 718	}
 719}
 720
 721static void clear_wq_table(void)
 722{
 723	int cpu;
 724
 725	for (cpu = 0; cpu < nr_cpus; cpu++)
 726		wq_table_clear_entry(cpu);
 727
 728	pr_debug("cleared wq table\n");
 729}
 730
 731static void free_iaa_device(struct iaa_device *iaa_device)
 732{
 733	if (!iaa_device)
 734		return;
 735
 736	remove_device_compression_modes(iaa_device);
 737	kfree(iaa_device);
 738}
 739
 740static void __free_iaa_wq(struct iaa_wq *iaa_wq)
 741{
 742	struct iaa_device *iaa_device;
 743
 744	if (!iaa_wq)
 745		return;
 746
 747	iaa_device = iaa_wq->iaa_device;
 748	if (iaa_device->n_wq == 0)
 749		free_iaa_device(iaa_wq->iaa_device);
 750}
 751
 752static void free_iaa_wq(struct iaa_wq *iaa_wq)
 753{
 754	struct idxd_wq *wq;
 755
 756	__free_iaa_wq(iaa_wq);
 757
 758	wq = iaa_wq->wq;
 759
 760	kfree(iaa_wq);
 761	idxd_wq_set_private(wq, NULL);
 762}
 763
 764static int iaa_wq_get(struct idxd_wq *wq)
 765{
 766	struct idxd_device *idxd = wq->idxd;
 767	struct iaa_wq *iaa_wq;
 768	int ret = 0;
 769
 770	spin_lock(&idxd->dev_lock);
 771	iaa_wq = idxd_wq_get_private(wq);
 772	if (iaa_wq && !iaa_wq->remove) {
 773		iaa_wq->ref++;
 774		idxd_wq_get(wq);
 775	} else {
 776		ret = -ENODEV;
 777	}
 778	spin_unlock(&idxd->dev_lock);
 779
 780	return ret;
 781}
 782
 783static int iaa_wq_put(struct idxd_wq *wq)
 784{
 785	struct idxd_device *idxd = wq->idxd;
 786	struct iaa_wq *iaa_wq;
 787	bool free = false;
 788	int ret = 0;
 789
 790	spin_lock(&idxd->dev_lock);
 791	iaa_wq = idxd_wq_get_private(wq);
 792	if (iaa_wq) {
 793		iaa_wq->ref--;
 794		if (iaa_wq->ref == 0 && iaa_wq->remove) {
 795			idxd_wq_set_private(wq, NULL);
 796			free = true;
 797		}
 798		idxd_wq_put(wq);
 799	} else {
 800		ret = -ENODEV;
 801	}
 802	spin_unlock(&idxd->dev_lock);
 803	if (free) {
 804		__free_iaa_wq(iaa_wq);
 805		kfree(iaa_wq);
 806	}
 807
 808	return ret;
 809}
 810
 811static void free_wq_table(void)
 812{
 813	int cpu;
 814
 815	for (cpu = 0; cpu < nr_cpus; cpu++)
 816		wq_table_free_entry(cpu);
 817
 818	free_percpu(wq_table);
 819
 820	pr_debug("freed wq table\n");
 821}
 822
 823static int alloc_wq_table(int max_wqs)
 824{
 825	struct wq_table_entry *entry;
 826	int cpu;
 827
 828	wq_table = alloc_percpu(struct wq_table_entry);
 829	if (!wq_table)
 830		return -ENOMEM;
 831
 832	for (cpu = 0; cpu < nr_cpus; cpu++) {
 833		entry = per_cpu_ptr(wq_table, cpu);
 834		entry->wqs = kcalloc(max_wqs, sizeof(struct wq *), GFP_KERNEL);
 835		if (!entry->wqs) {
 836			free_wq_table();
 837			return -ENOMEM;
 838		}
 839
 840		entry->max_wqs = max_wqs;
 841	}
 842
 843	pr_debug("initialized wq table\n");
 844
 845	return 0;
 846}
 847
 848static int save_iaa_wq(struct idxd_wq *wq)
 849{
 850	struct iaa_device *iaa_device, *found = NULL;
 851	struct idxd_device *idxd;
 852	struct pci_dev *pdev;
 853	struct device *dev;
 854	int ret = 0;
 855
 856	list_for_each_entry(iaa_device, &iaa_devices, list) {
 857		if (iaa_device->idxd == wq->idxd) {
 858			idxd = iaa_device->idxd;
 859			pdev = idxd->pdev;
 860			dev = &pdev->dev;
 861			/*
 862			 * Check to see that we don't already have this wq.
 863			 * Shouldn't happen but we don't control probing.
 864			 */
 865			if (iaa_has_wq(iaa_device, wq)) {
 866				dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n",
 867					iaa_device);
 868				goto out;
 869			}
 870
 871			found = iaa_device;
 872
 873			ret = add_iaa_wq(iaa_device, wq, NULL);
 874			if (ret)
 875				goto out;
 876
 877			break;
 878		}
 879	}
 880
 881	if (!found) {
 882		struct iaa_device *new_device;
 883		struct iaa_wq *new_wq;
 884
 885		new_device = add_iaa_device(wq->idxd);
 886		if (!new_device) {
 887			ret = -ENOMEM;
 888			goto out;
 889		}
 890
 891		ret = add_iaa_wq(new_device, wq, &new_wq);
 892		if (ret) {
 893			del_iaa_device(new_device);
 894			free_iaa_device(new_device);
 895			goto out;
 896		}
 897
 898		ret = init_iaa_device(new_device, new_wq);
 899		if (ret) {
 900			del_iaa_wq(new_device, new_wq->wq);
 901			del_iaa_device(new_device);
 902			free_iaa_wq(new_wq);
 903			goto out;
 904		}
 905	}
 906
 907	if (WARN_ON(nr_iaa == 0))
 908		return -EINVAL;
 909
 910	cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
 
 
 911out:
 912	return 0;
 913}
 914
 915static void remove_iaa_wq(struct idxd_wq *wq)
 916{
 917	struct iaa_device *iaa_device;
 918
 919	list_for_each_entry(iaa_device, &iaa_devices, list) {
 920		if (iaa_has_wq(iaa_device, wq)) {
 921			del_iaa_wq(iaa_device, wq);
 922			break;
 923		}
 924	}
 925
 926	if (nr_iaa)
 927		cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
 928	else
 929		cpus_per_iaa = 0;
 
 
 930}
 931
 932static int wq_table_add_wqs(int iaa, int cpu)
 933{
 934	struct iaa_device *iaa_device, *found_device = NULL;
 935	int ret = 0, cur_iaa = 0, n_wqs_added = 0;
 936	struct idxd_device *idxd;
 937	struct iaa_wq *iaa_wq;
 938	struct pci_dev *pdev;
 939	struct device *dev;
 940
 941	list_for_each_entry(iaa_device, &iaa_devices, list) {
 942		idxd = iaa_device->idxd;
 943		pdev = idxd->pdev;
 944		dev = &pdev->dev;
 945
 946		if (cur_iaa != iaa) {
 947			cur_iaa++;
 948			continue;
 949		}
 950
 951		found_device = iaa_device;
 952		dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n",
 953			found_device->idxd->id, cur_iaa);
 954		break;
 955	}
 956
 957	if (!found_device) {
 958		found_device = list_first_entry_or_null(&iaa_devices,
 959							struct iaa_device, list);
 960		if (!found_device) {
 961			pr_debug("couldn't find any iaa devices with wqs!\n");
 962			ret = -EINVAL;
 963			goto out;
 964		}
 965		cur_iaa = 0;
 966
 967		idxd = found_device->idxd;
 968		pdev = idxd->pdev;
 969		dev = &pdev->dev;
 970		dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n",
 971			found_device->idxd->id, cur_iaa);
 972	}
 973
 974	list_for_each_entry(iaa_wq, &found_device->wqs, list) {
 975		wq_table_add(cpu, iaa_wq->wq);
 976		pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n",
 977			 cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id);
 978		n_wqs_added++;
 979	}
 980
 981	if (!n_wqs_added) {
 982		pr_debug("couldn't find any iaa wqs!\n");
 983		ret = -EINVAL;
 984		goto out;
 985	}
 986out:
 987	return ret;
 988}
 989
 990/*
 991 * Rebalance the wq table so that given a cpu, it's easy to find the
 992 * closest IAA instance.  The idea is to try to choose the most
 993 * appropriate IAA instance for a caller and spread available
 994 * workqueues around to clients.
 995 */
 996static void rebalance_wq_table(void)
 997{
 998	const struct cpumask *node_cpus;
 999	int node, cpu, iaa = -1;
1000
1001	if (nr_iaa == 0)
1002		return;
1003
1004	pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n",
1005		 nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa);
1006
1007	clear_wq_table();
1008
1009	if (nr_iaa == 1) {
1010		for (cpu = 0; cpu < nr_cpus; cpu++) {
1011			if (WARN_ON(wq_table_add_wqs(0, cpu))) {
1012				pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu);
1013				return;
1014			}
1015		}
1016
1017		return;
1018	}
1019
1020	for_each_node_with_cpus(node) {
1021		node_cpus = cpumask_of_node(node);
1022
1023		for (cpu = 0; cpu < nr_cpus_per_node; cpu++) {
1024			int node_cpu = cpumask_nth(cpu, node_cpus);
1025
1026			if (WARN_ON(node_cpu >= nr_cpu_ids)) {
1027				pr_debug("node_cpu %d doesn't exist!\n", node_cpu);
1028				return;
1029			}
1030
1031			if ((cpu % cpus_per_iaa) == 0)
1032				iaa++;
1033
1034			if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) {
1035				pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu);
1036				return;
1037			}
1038		}
1039	}
1040}
1041
1042static inline int check_completion(struct device *dev,
1043				   struct iax_completion_record *comp,
1044				   bool compress,
1045				   bool only_once)
1046{
1047	char *op_str = compress ? "compress" : "decompress";
1048	int ret = 0;
1049
1050	while (!comp->status) {
1051		if (only_once)
1052			return -EAGAIN;
1053		cpu_relax();
1054	}
1055
1056	if (comp->status != IAX_COMP_SUCCESS) {
1057		if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) {
1058			ret = -ETIMEDOUT;
1059			dev_dbg(dev, "%s timed out, size=0x%x\n",
1060				op_str, comp->output_size);
1061			update_completion_timeout_errs();
1062			goto out;
1063		}
1064
1065		if (comp->status == IAA_ANALYTICS_ERROR &&
1066		    comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) {
1067			ret = -E2BIG;
1068			dev_dbg(dev, "compressed > uncompressed size,"
1069				" not compressing, size=0x%x\n",
1070				comp->output_size);
1071			update_completion_comp_buf_overflow_errs();
1072			goto out;
1073		}
1074
1075		if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) {
1076			ret = -EOVERFLOW;
1077			goto out;
1078		}
1079
1080		ret = -EINVAL;
1081		dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n",
1082			op_str, comp->status, comp->error_code, comp->output_size);
1083		print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, comp, 64, 0);
1084		update_completion_einval_errs();
1085
1086		goto out;
1087	}
1088out:
1089	return ret;
1090}
1091
1092static int deflate_generic_decompress(struct acomp_req *req)
1093{
1094	void *src, *dst;
1095	int ret;
1096
1097	src = kmap_local_page(sg_page(req->src)) + req->src->offset;
1098	dst = kmap_local_page(sg_page(req->dst)) + req->dst->offset;
1099
1100	ret = crypto_comp_decompress(deflate_generic_tfm,
1101				     src, req->slen, dst, &req->dlen);
1102
1103	kunmap_local(src);
1104	kunmap_local(dst);
1105
1106	update_total_sw_decomp_calls();
1107
1108	return ret;
1109}
1110
1111static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1112				struct acomp_req *req,
1113				dma_addr_t *src_addr, dma_addr_t *dst_addr);
1114
1115static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1116			       struct idxd_wq *wq,
1117			       dma_addr_t src_addr, unsigned int slen,
1118			       dma_addr_t dst_addr, unsigned int *dlen,
1119			       u32 compression_crc);
1120
1121static void iaa_desc_complete(struct idxd_desc *idxd_desc,
1122			      enum idxd_complete_type comp_type,
1123			      bool free_desc, void *__ctx,
1124			      u32 *status)
1125{
1126	struct iaa_device_compression_mode *active_compression_mode;
1127	struct iaa_compression_ctx *compression_ctx;
1128	struct crypto_ctx *ctx = __ctx;
1129	struct iaa_device *iaa_device;
1130	struct idxd_device *idxd;
1131	struct iaa_wq *iaa_wq;
1132	struct pci_dev *pdev;
1133	struct device *dev;
1134	int ret, err = 0;
1135
1136	compression_ctx = crypto_tfm_ctx(ctx->tfm);
1137
1138	iaa_wq = idxd_wq_get_private(idxd_desc->wq);
1139	iaa_device = iaa_wq->iaa_device;
1140	idxd = iaa_device->idxd;
1141	pdev = idxd->pdev;
1142	dev = &pdev->dev;
1143
1144	active_compression_mode = get_iaa_device_compression_mode(iaa_device,
1145								  compression_ctx->mode);
1146	dev_dbg(dev, "%s: compression mode %s,"
1147		" ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__,
1148		active_compression_mode->name,
1149		ctx->src_addr, ctx->dst_addr);
1150
1151	ret = check_completion(dev, idxd_desc->iax_completion,
1152			       ctx->compress, false);
1153	if (ret) {
1154		dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1155		if (!ctx->compress &&
1156		    idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1157			pr_warn("%s: falling back to deflate-generic decompress, "
1158				"analytics error code %x\n", __func__,
1159				idxd_desc->iax_completion->error_code);
1160			ret = deflate_generic_decompress(ctx->req);
1161			if (ret) {
1162				dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1163					__func__, ret);
1164				err = -EIO;
1165				goto err;
1166			}
1167		} else {
1168			err = -EIO;
1169			goto err;
1170		}
1171	} else {
1172		ctx->req->dlen = idxd_desc->iax_completion->output_size;
1173	}
1174
1175	/* Update stats */
1176	if (ctx->compress) {
1177		update_total_comp_bytes_out(ctx->req->dlen);
1178		update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen);
1179	} else {
1180		update_total_decomp_bytes_in(ctx->req->dlen);
1181		update_wq_decomp_bytes(iaa_wq->wq, ctx->req->dlen);
1182	}
1183
1184	if (ctx->compress && compression_ctx->verify_compress) {
1185		dma_addr_t src_addr, dst_addr;
1186		u32 compression_crc;
1187
1188		compression_crc = idxd_desc->iax_completion->crc;
1189
1190		ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);
1191		if (ret) {
1192			dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1193			err = -EIO;
1194			goto out;
1195		}
1196
1197		ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr,
1198					  ctx->req->slen, dst_addr, &ctx->req->dlen,
1199					  compression_crc);
1200		if (ret) {
1201			dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret);
1202			err = -EIO;
1203		}
1204
1205		dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE);
1206		dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE);
1207
1208		goto out;
1209	}
1210err:
1211	dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
1212	dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
1213out:
1214	if (ret != 0)
1215		dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1216
1217	if (ctx->req->base.complete)
1218		acomp_request_complete(ctx->req, err);
1219
1220	if (free_desc)
1221		idxd_free_desc(idxd_desc->wq, idxd_desc);
1222	iaa_wq_put(idxd_desc->wq);
1223}
1224
1225static int iaa_compress(struct crypto_tfm *tfm,	struct acomp_req *req,
1226			struct idxd_wq *wq,
1227			dma_addr_t src_addr, unsigned int slen,
1228			dma_addr_t dst_addr, unsigned int *dlen,
1229			u32 *compression_crc,
1230			bool disable_async)
1231{
1232	struct iaa_device_compression_mode *active_compression_mode;
1233	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1234	struct iaa_device *iaa_device;
1235	struct idxd_desc *idxd_desc;
1236	struct iax_hw_desc *desc;
1237	struct idxd_device *idxd;
1238	struct iaa_wq *iaa_wq;
1239	struct pci_dev *pdev;
1240	struct device *dev;
1241	int ret = 0;
1242
1243	iaa_wq = idxd_wq_get_private(wq);
1244	iaa_device = iaa_wq->iaa_device;
1245	idxd = iaa_device->idxd;
1246	pdev = idxd->pdev;
1247	dev = &pdev->dev;
1248
1249	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1250
1251	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1252	if (IS_ERR(idxd_desc)) {
1253		dev_dbg(dev, "idxd descriptor allocation failed\n");
1254		dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc));
1255		return PTR_ERR(idxd_desc);
1256	}
1257	desc = idxd_desc->iax_hw;
1258
1259	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR |
1260		IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC;
1261	desc->opcode = IAX_OPCODE_COMPRESS;
1262	desc->compr_flags = IAA_COMP_FLAGS;
1263	desc->priv = 0;
1264
1265	desc->src1_addr = (u64)src_addr;
1266	desc->src1_size = slen;
1267	desc->dst_addr = (u64)dst_addr;
1268	desc->max_dst_size = *dlen;
1269	desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr;
1270	desc->src2_size = sizeof(struct aecs_comp_table_record);
1271	desc->completion_addr = idxd_desc->compl_dma;
1272
1273	if (ctx->use_irq && !disable_async) {
1274		desc->flags |= IDXD_OP_FLAG_RCI;
1275
1276		idxd_desc->crypto.req = req;
1277		idxd_desc->crypto.tfm = tfm;
1278		idxd_desc->crypto.src_addr = src_addr;
1279		idxd_desc->crypto.dst_addr = dst_addr;
1280		idxd_desc->crypto.compress = true;
1281
1282		dev_dbg(dev, "%s use_async_irq: compression mode %s,"
1283			" src_addr %llx, dst_addr %llx\n", __func__,
1284			active_compression_mode->name,
1285			src_addr, dst_addr);
1286	} else if (ctx->async_mode && !disable_async)
1287		req->base.data = idxd_desc;
1288
1289	dev_dbg(dev, "%s: compression mode %s,"
1290		" desc->src1_addr %llx, desc->src1_size %d,"
1291		" desc->dst_addr %llx, desc->max_dst_size %d,"
1292		" desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1293		active_compression_mode->name,
1294		desc->src1_addr, desc->src1_size, desc->dst_addr,
1295		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1296
1297	ret = idxd_submit_desc(wq, idxd_desc);
1298	if (ret) {
1299		dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1300		goto err;
1301	}
1302
1303	/* Update stats */
1304	update_total_comp_calls();
1305	update_wq_comp_calls(wq);
1306
1307	if (ctx->async_mode && !disable_async) {
1308		ret = -EINPROGRESS;
1309		dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1310		goto out;
1311	}
1312
1313	ret = check_completion(dev, idxd_desc->iax_completion, true, false);
1314	if (ret) {
1315		dev_dbg(dev, "check_completion failed ret=%d\n", ret);
1316		goto err;
1317	}
1318
1319	*dlen = idxd_desc->iax_completion->output_size;
1320
1321	/* Update stats */
1322	update_total_comp_bytes_out(*dlen);
1323	update_wq_comp_bytes(wq, *dlen);
1324
1325	*compression_crc = idxd_desc->iax_completion->crc;
1326
1327	if (!ctx->async_mode)
1328		idxd_free_desc(wq, idxd_desc);
1329out:
1330	return ret;
1331err:
1332	idxd_free_desc(wq, idxd_desc);
1333	dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1334
1335	goto out;
1336}
1337
1338static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1339				struct acomp_req *req,
1340				dma_addr_t *src_addr, dma_addr_t *dst_addr)
1341{
1342	int ret = 0;
1343	int nr_sgs;
1344
1345	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1346	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1347
1348	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1349	if (nr_sgs <= 0 || nr_sgs > 1) {
1350		dev_dbg(dev, "verify: couldn't map src sg for iaa device %d,"
1351			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1352			iaa_wq->wq->id, ret);
1353		ret = -EIO;
1354		goto out;
1355	}
1356	*src_addr = sg_dma_address(req->src);
1357	dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1358		" req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs,
1359		req->src, req->slen, sg_dma_len(req->src));
1360
1361	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1362	if (nr_sgs <= 0 || nr_sgs > 1) {
1363		dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d,"
1364			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1365			iaa_wq->wq->id, ret);
1366		ret = -EIO;
1367		dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1368		goto out;
1369	}
1370	*dst_addr = sg_dma_address(req->dst);
1371	dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1372		" req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs,
1373		req->dst, req->dlen, sg_dma_len(req->dst));
1374out:
1375	return ret;
1376}
1377
1378static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1379			       struct idxd_wq *wq,
1380			       dma_addr_t src_addr, unsigned int slen,
1381			       dma_addr_t dst_addr, unsigned int *dlen,
1382			       u32 compression_crc)
1383{
1384	struct iaa_device_compression_mode *active_compression_mode;
1385	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1386	struct iaa_device *iaa_device;
1387	struct idxd_desc *idxd_desc;
1388	struct iax_hw_desc *desc;
1389	struct idxd_device *idxd;
1390	struct iaa_wq *iaa_wq;
1391	struct pci_dev *pdev;
1392	struct device *dev;
1393	int ret = 0;
1394
1395	iaa_wq = idxd_wq_get_private(wq);
1396	iaa_device = iaa_wq->iaa_device;
1397	idxd = iaa_device->idxd;
1398	pdev = idxd->pdev;
1399	dev = &pdev->dev;
1400
1401	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1402
1403	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1404	if (IS_ERR(idxd_desc)) {
1405		dev_dbg(dev, "idxd descriptor allocation failed\n");
1406		dev_dbg(dev, "iaa compress failed: ret=%ld\n",
1407			PTR_ERR(idxd_desc));
1408		return PTR_ERR(idxd_desc);
1409	}
1410	desc = idxd_desc->iax_hw;
1411
1412	/* Verify (optional) - decompress and check crc, suppress dest write */
1413
1414	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1415	desc->opcode = IAX_OPCODE_DECOMPRESS;
1416	desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT;
1417	desc->priv = 0;
1418
1419	desc->src1_addr = (u64)dst_addr;
1420	desc->src1_size = *dlen;
1421	desc->dst_addr = (u64)src_addr;
1422	desc->max_dst_size = slen;
1423	desc->completion_addr = idxd_desc->compl_dma;
1424
1425	dev_dbg(dev, "(verify) compression mode %s,"
1426		" desc->src1_addr %llx, desc->src1_size %d,"
1427		" desc->dst_addr %llx, desc->max_dst_size %d,"
1428		" desc->src2_addr %llx, desc->src2_size %d\n",
1429		active_compression_mode->name,
1430		desc->src1_addr, desc->src1_size, desc->dst_addr,
1431		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1432
1433	ret = idxd_submit_desc(wq, idxd_desc);
1434	if (ret) {
1435		dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret);
1436		goto err;
1437	}
1438
1439	ret = check_completion(dev, idxd_desc->iax_completion, false, false);
1440	if (ret) {
1441		dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret);
1442		goto err;
1443	}
1444
1445	if (compression_crc != idxd_desc->iax_completion->crc) {
1446		ret = -EINVAL;
1447		dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:"
1448			" comp=0x%x, decomp=0x%x\n", compression_crc,
1449			idxd_desc->iax_completion->crc);
1450		print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET,
1451			       8, 1, idxd_desc->iax_completion, 64, 0);
1452		goto err;
1453	}
1454
1455	idxd_free_desc(wq, idxd_desc);
1456out:
1457	return ret;
1458err:
1459	idxd_free_desc(wq, idxd_desc);
1460	dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1461
1462	goto out;
1463}
1464
1465static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
1466			  struct idxd_wq *wq,
1467			  dma_addr_t src_addr, unsigned int slen,
1468			  dma_addr_t dst_addr, unsigned int *dlen,
1469			  bool disable_async)
1470{
1471	struct iaa_device_compression_mode *active_compression_mode;
1472	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1473	struct iaa_device *iaa_device;
1474	struct idxd_desc *idxd_desc;
1475	struct iax_hw_desc *desc;
1476	struct idxd_device *idxd;
1477	struct iaa_wq *iaa_wq;
1478	struct pci_dev *pdev;
1479	struct device *dev;
1480	int ret = 0;
1481
1482	iaa_wq = idxd_wq_get_private(wq);
1483	iaa_device = iaa_wq->iaa_device;
1484	idxd = iaa_device->idxd;
1485	pdev = idxd->pdev;
1486	dev = &pdev->dev;
1487
1488	active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1489
1490	idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1491	if (IS_ERR(idxd_desc)) {
1492		dev_dbg(dev, "idxd descriptor allocation failed\n");
1493		dev_dbg(dev, "iaa decompress failed: ret=%ld\n",
1494			PTR_ERR(idxd_desc));
1495		return PTR_ERR(idxd_desc);
1496	}
1497	desc = idxd_desc->iax_hw;
1498
1499	desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1500	desc->opcode = IAX_OPCODE_DECOMPRESS;
1501	desc->max_dst_size = PAGE_SIZE;
1502	desc->decompr_flags = IAA_DECOMP_FLAGS;
1503	desc->priv = 0;
1504
1505	desc->src1_addr = (u64)src_addr;
1506	desc->dst_addr = (u64)dst_addr;
1507	desc->max_dst_size = *dlen;
1508	desc->src1_size = slen;
1509	desc->completion_addr = idxd_desc->compl_dma;
1510
1511	if (ctx->use_irq && !disable_async) {
1512		desc->flags |= IDXD_OP_FLAG_RCI;
1513
1514		idxd_desc->crypto.req = req;
1515		idxd_desc->crypto.tfm = tfm;
1516		idxd_desc->crypto.src_addr = src_addr;
1517		idxd_desc->crypto.dst_addr = dst_addr;
1518		idxd_desc->crypto.compress = false;
1519
1520		dev_dbg(dev, "%s: use_async_irq compression mode %s,"
1521			" src_addr %llx, dst_addr %llx\n", __func__,
1522			active_compression_mode->name,
1523			src_addr, dst_addr);
1524	} else if (ctx->async_mode && !disable_async)
1525		req->base.data = idxd_desc;
1526
1527	dev_dbg(dev, "%s: decompression mode %s,"
1528		" desc->src1_addr %llx, desc->src1_size %d,"
1529		" desc->dst_addr %llx, desc->max_dst_size %d,"
1530		" desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1531		active_compression_mode->name,
1532		desc->src1_addr, desc->src1_size, desc->dst_addr,
1533		desc->max_dst_size, desc->src2_addr, desc->src2_size);
1534
1535	ret = idxd_submit_desc(wq, idxd_desc);
1536	if (ret) {
1537		dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1538		goto err;
1539	}
1540
1541	/* Update stats */
1542	update_total_decomp_calls();
1543	update_wq_decomp_calls(wq);
1544
1545	if (ctx->async_mode && !disable_async) {
1546		ret = -EINPROGRESS;
1547		dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1548		goto out;
1549	}
1550
1551	ret = check_completion(dev, idxd_desc->iax_completion, false, false);
1552	if (ret) {
1553		dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1554		if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1555			pr_warn("%s: falling back to deflate-generic decompress, "
1556				"analytics error code %x\n", __func__,
1557				idxd_desc->iax_completion->error_code);
1558			ret = deflate_generic_decompress(req);
1559			if (ret) {
1560				dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1561					__func__, ret);
1562				goto err;
1563			}
1564		} else {
1565			goto err;
1566		}
1567	} else {
1568		req->dlen = idxd_desc->iax_completion->output_size;
1569	}
1570
1571	*dlen = req->dlen;
1572
1573	if (!ctx->async_mode)
1574		idxd_free_desc(wq, idxd_desc);
1575
1576	/* Update stats */
1577	update_total_decomp_bytes_in(slen);
1578	update_wq_decomp_bytes(wq, slen);
1579out:
1580	return ret;
1581err:
1582	idxd_free_desc(wq, idxd_desc);
1583	dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret);
1584
1585	goto out;
1586}
1587
1588static int iaa_comp_acompress(struct acomp_req *req)
1589{
1590	struct iaa_compression_ctx *compression_ctx;
1591	struct crypto_tfm *tfm = req->base.tfm;
1592	dma_addr_t src_addr, dst_addr;
1593	bool disable_async = false;
1594	int nr_sgs, cpu, ret = 0;
1595	struct iaa_wq *iaa_wq;
1596	u32 compression_crc;
1597	struct idxd_wq *wq;
1598	struct device *dev;
 
1599	int order = -1;
1600
1601	compression_ctx = crypto_tfm_ctx(tfm);
1602
1603	if (!iaa_crypto_enabled) {
1604		pr_debug("iaa_crypto disabled, not compressing\n");
1605		return -ENODEV;
1606	}
1607
1608	if (!req->src || !req->slen) {
1609		pr_debug("invalid src, not compressing\n");
1610		return -EINVAL;
1611	}
1612
1613	cpu = get_cpu();
1614	wq = wq_table_next_wq(cpu);
1615	put_cpu();
1616	if (!wq) {
1617		pr_debug("no wq configured for cpu=%d\n", cpu);
1618		return -ENODEV;
1619	}
1620
1621	ret = iaa_wq_get(wq);
1622	if (ret) {
1623		pr_debug("no wq available for cpu=%d\n", cpu);
1624		return -ENODEV;
1625	}
1626
1627	iaa_wq = idxd_wq_get_private(wq);
1628
1629	if (!req->dst) {
1630		gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
1631
1632		/* incompressible data will always be < 2 * slen */
1633		req->dlen = 2 * req->slen;
1634		order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE);
1635		req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL);
1636		if (!req->dst) {
1637			ret = -ENOMEM;
1638			order = -1;
1639			goto out;
1640		}
1641		disable_async = true;
1642	}
1643
1644	dev = &wq->idxd->pdev->dev;
1645
1646	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1647	if (nr_sgs <= 0 || nr_sgs > 1) {
1648		dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1649			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1650			iaa_wq->wq->id, ret);
1651		ret = -EIO;
1652		goto out;
1653	}
1654	src_addr = sg_dma_address(req->src);
1655	dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1656		" req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1657		req->src, req->slen, sg_dma_len(req->src));
1658
1659	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1660	if (nr_sgs <= 0 || nr_sgs > 1) {
1661		dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1662			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1663			iaa_wq->wq->id, ret);
1664		ret = -EIO;
1665		goto err_map_dst;
1666	}
1667	dst_addr = sg_dma_address(req->dst);
1668	dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1669		" req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1670		req->dst, req->dlen, sg_dma_len(req->dst));
1671
 
1672	ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr,
1673			   &req->dlen, &compression_crc, disable_async);
 
1674	if (ret == -EINPROGRESS)
1675		return ret;
1676
1677	if (!ret && compression_ctx->verify_compress) {
1678		ret = iaa_remap_for_verify(dev, iaa_wq, req, &src_addr, &dst_addr);
1679		if (ret) {
1680			dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1681			goto out;
1682		}
1683
1684		ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen,
1685					  dst_addr, &req->dlen, compression_crc);
1686		if (ret)
1687			dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret);
1688
1689		dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1690		dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1691
1692		goto out;
1693	}
1694
1695	if (ret)
1696		dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1697
1698	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1699err_map_dst:
1700	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1701out:
1702	iaa_wq_put(wq);
1703
1704	if (order >= 0)
1705		sgl_free_order(req->dst, order);
1706
1707	return ret;
1708}
1709
1710static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req)
1711{
1712	gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1713		GFP_KERNEL : GFP_ATOMIC;
1714	struct crypto_tfm *tfm = req->base.tfm;
1715	dma_addr_t src_addr, dst_addr;
1716	int nr_sgs, cpu, ret = 0;
1717	struct iaa_wq *iaa_wq;
1718	struct device *dev;
1719	struct idxd_wq *wq;
 
1720	int order = -1;
1721
1722	cpu = get_cpu();
1723	wq = wq_table_next_wq(cpu);
1724	put_cpu();
1725	if (!wq) {
1726		pr_debug("no wq configured for cpu=%d\n", cpu);
1727		return -ENODEV;
1728	}
1729
1730	ret = iaa_wq_get(wq);
1731	if (ret) {
1732		pr_debug("no wq available for cpu=%d\n", cpu);
1733		return -ENODEV;
1734	}
1735
1736	iaa_wq = idxd_wq_get_private(wq);
1737
1738	dev = &wq->idxd->pdev->dev;
1739
1740	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1741	if (nr_sgs <= 0 || nr_sgs > 1) {
1742		dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1743			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1744			iaa_wq->wq->id, ret);
1745		ret = -EIO;
1746		goto out;
1747	}
1748	src_addr = sg_dma_address(req->src);
1749	dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1750		" req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1751		req->src, req->slen, sg_dma_len(req->src));
1752
1753	req->dlen = 4 * req->slen; /* start with ~avg comp rato */
1754alloc_dest:
1755	order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE);
1756	req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL);
1757	if (!req->dst) {
1758		ret = -ENOMEM;
1759		order = -1;
1760		goto out;
1761	}
1762
1763	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1764	if (nr_sgs <= 0 || nr_sgs > 1) {
1765		dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1766			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1767			iaa_wq->wq->id, ret);
1768		ret = -EIO;
1769		goto err_map_dst;
1770	}
1771
1772	dst_addr = sg_dma_address(req->dst);
1773	dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1774		" req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1775		req->dst, req->dlen, sg_dma_len(req->dst));
 
1776	ret = iaa_decompress(tfm, req, wq, src_addr, req->slen,
1777			     dst_addr, &req->dlen, true);
 
1778	if (ret == -EOVERFLOW) {
1779		dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1780		req->dlen *= 2;
1781		if (req->dlen > CRYPTO_ACOMP_DST_MAX)
1782			goto err_map_dst;
1783		goto alloc_dest;
1784	}
1785
1786	if (ret != 0)
1787		dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
1788
1789	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1790err_map_dst:
1791	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1792out:
1793	iaa_wq_put(wq);
1794
1795	if (order >= 0)
1796		sgl_free_order(req->dst, order);
1797
1798	return ret;
1799}
1800
1801static int iaa_comp_adecompress(struct acomp_req *req)
1802{
1803	struct crypto_tfm *tfm = req->base.tfm;
1804	dma_addr_t src_addr, dst_addr;
1805	int nr_sgs, cpu, ret = 0;
1806	struct iaa_wq *iaa_wq;
1807	struct device *dev;
 
1808	struct idxd_wq *wq;
1809
1810	if (!iaa_crypto_enabled) {
1811		pr_debug("iaa_crypto disabled, not decompressing\n");
1812		return -ENODEV;
1813	}
1814
1815	if (!req->src || !req->slen) {
1816		pr_debug("invalid src, not decompressing\n");
1817		return -EINVAL;
1818	}
1819
1820	if (!req->dst)
1821		return iaa_comp_adecompress_alloc_dest(req);
1822
1823	cpu = get_cpu();
1824	wq = wq_table_next_wq(cpu);
1825	put_cpu();
1826	if (!wq) {
1827		pr_debug("no wq configured for cpu=%d\n", cpu);
1828		return -ENODEV;
1829	}
1830
1831	ret = iaa_wq_get(wq);
1832	if (ret) {
1833		pr_debug("no wq available for cpu=%d\n", cpu);
1834		return -ENODEV;
1835	}
1836
1837	iaa_wq = idxd_wq_get_private(wq);
1838
1839	dev = &wq->idxd->pdev->dev;
1840
1841	nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1842	if (nr_sgs <= 0 || nr_sgs > 1) {
1843		dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1844			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1845			iaa_wq->wq->id, ret);
1846		ret = -EIO;
1847		goto out;
1848	}
1849	src_addr = sg_dma_address(req->src);
1850	dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1851		" req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1852		req->src, req->slen, sg_dma_len(req->src));
1853
1854	nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1855	if (nr_sgs <= 0 || nr_sgs > 1) {
1856		dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1857			" wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1858			iaa_wq->wq->id, ret);
1859		ret = -EIO;
1860		goto err_map_dst;
1861	}
1862	dst_addr = sg_dma_address(req->dst);
1863	dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1864		" req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1865		req->dst, req->dlen, sg_dma_len(req->dst));
1866
 
1867	ret = iaa_decompress(tfm, req, wq, src_addr, req->slen,
1868			     dst_addr, &req->dlen, false);
 
1869	if (ret == -EINPROGRESS)
1870		return ret;
1871
1872	if (ret != 0)
1873		dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
1874
1875	dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1876err_map_dst:
1877	dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1878out:
1879	iaa_wq_put(wq);
1880
1881	return ret;
1882}
1883
1884static void compression_ctx_init(struct iaa_compression_ctx *ctx)
1885{
1886	ctx->verify_compress = iaa_verify_compress;
1887	ctx->async_mode = async_mode;
1888	ctx->use_irq = use_irq;
1889}
1890
1891static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm)
1892{
1893	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
1894	struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1895
1896	compression_ctx_init(ctx);
1897
1898	ctx->mode = IAA_MODE_FIXED;
1899
1900	return 0;
1901}
1902
1903static void dst_free(struct scatterlist *sgl)
1904{
1905	/*
1906	 * Called for req->dst = NULL cases but we free elsewhere
1907	 * using sgl_free_order().
1908	 */
1909}
1910
1911static struct acomp_alg iaa_acomp_fixed_deflate = {
1912	.init			= iaa_comp_init_fixed,
1913	.compress		= iaa_comp_acompress,
1914	.decompress		= iaa_comp_adecompress,
1915	.dst_free               = dst_free,
1916	.base			= {
1917		.cra_name		= "deflate",
1918		.cra_driver_name	= "deflate-iaa",
 
1919		.cra_ctxsize		= sizeof(struct iaa_compression_ctx),
1920		.cra_module		= THIS_MODULE,
1921		.cra_priority		= IAA_ALG_PRIORITY,
1922	}
1923};
1924
1925static int iaa_register_compression_device(void)
1926{
1927	int ret;
1928
1929	ret = crypto_register_acomp(&iaa_acomp_fixed_deflate);
1930	if (ret) {
1931		pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret);
1932		goto out;
1933	}
1934
1935	iaa_crypto_registered = true;
1936out:
1937	return ret;
1938}
1939
1940static int iaa_unregister_compression_device(void)
1941{
1942	if (iaa_crypto_registered)
1943		crypto_unregister_acomp(&iaa_acomp_fixed_deflate);
1944
1945	return 0;
1946}
1947
1948static int iaa_crypto_probe(struct idxd_dev *idxd_dev)
1949{
1950	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1951	struct idxd_device *idxd = wq->idxd;
1952	struct idxd_driver_data *data = idxd->data;
1953	struct device *dev = &idxd_dev->conf_dev;
1954	bool first_wq = false;
1955	int ret = 0;
1956
1957	if (idxd->state != IDXD_DEV_ENABLED)
1958		return -ENXIO;
1959
1960	if (data->type != IDXD_TYPE_IAX)
1961		return -ENODEV;
1962
1963	mutex_lock(&wq->wq_lock);
1964
1965	if (idxd_wq_get_private(wq)) {
1966		mutex_unlock(&wq->wq_lock);
1967		return -EBUSY;
1968	}
1969
1970	if (!idxd_wq_driver_name_match(wq, dev)) {
1971		dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n",
1972			idxd->id, wq->id, wq->driver_name, dev->driver->name);
1973		idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
1974		ret = -ENODEV;
1975		goto err;
1976	}
1977
1978	wq->type = IDXD_WQT_KERNEL;
1979
1980	ret = idxd_drv_enable_wq(wq);
1981	if (ret < 0) {
1982		dev_dbg(dev, "enable wq %d.%d failed: %d\n",
1983			idxd->id, wq->id, ret);
1984		ret = -ENXIO;
1985		goto err;
1986	}
1987
1988	mutex_lock(&iaa_devices_lock);
1989
1990	if (list_empty(&iaa_devices)) {
1991		ret = alloc_wq_table(wq->idxd->max_wqs);
1992		if (ret)
1993			goto err_alloc;
1994		first_wq = true;
1995	}
1996
1997	ret = save_iaa_wq(wq);
1998	if (ret)
1999		goto err_save;
2000
2001	rebalance_wq_table();
2002
2003	if (first_wq) {
2004		iaa_crypto_enabled = true;
2005		ret = iaa_register_compression_device();
2006		if (ret != 0) {
2007			iaa_crypto_enabled = false;
2008			dev_dbg(dev, "IAA compression device registration failed\n");
2009			goto err_register;
2010		}
2011		try_module_get(THIS_MODULE);
2012
2013		pr_info("iaa_crypto now ENABLED\n");
2014	}
2015
2016	mutex_unlock(&iaa_devices_lock);
2017out:
2018	mutex_unlock(&wq->wq_lock);
2019
2020	return ret;
2021
2022err_register:
2023	remove_iaa_wq(wq);
2024	free_iaa_wq(idxd_wq_get_private(wq));
2025err_save:
2026	if (first_wq)
2027		free_wq_table();
2028err_alloc:
2029	mutex_unlock(&iaa_devices_lock);
2030	idxd_drv_disable_wq(wq);
2031err:
2032	wq->type = IDXD_WQT_NONE;
2033
2034	goto out;
2035}
2036
2037static void iaa_crypto_remove(struct idxd_dev *idxd_dev)
2038{
2039	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
2040	struct idxd_device *idxd = wq->idxd;
2041	struct iaa_wq *iaa_wq;
2042	bool free = false;
2043
2044	idxd_wq_quiesce(wq);
2045
2046	mutex_lock(&wq->wq_lock);
2047	mutex_lock(&iaa_devices_lock);
2048
2049	remove_iaa_wq(wq);
2050
2051	spin_lock(&idxd->dev_lock);
2052	iaa_wq = idxd_wq_get_private(wq);
2053	if (!iaa_wq) {
2054		spin_unlock(&idxd->dev_lock);
2055		pr_err("%s: no iaa_wq available to remove\n", __func__);
2056		goto out;
2057	}
2058
2059	if (iaa_wq->ref) {
2060		iaa_wq->remove = true;
2061	} else {
2062		wq = iaa_wq->wq;
2063		idxd_wq_set_private(wq, NULL);
2064		free = true;
2065	}
2066	spin_unlock(&idxd->dev_lock);
2067	if (free) {
2068		__free_iaa_wq(iaa_wq);
2069		kfree(iaa_wq);
2070	}
2071
2072	idxd_drv_disable_wq(wq);
2073	rebalance_wq_table();
2074
2075	if (nr_iaa == 0) {
2076		iaa_crypto_enabled = false;
2077		free_wq_table();
2078		module_put(THIS_MODULE);
2079
2080		pr_info("iaa_crypto now DISABLED\n");
2081	}
2082out:
2083	mutex_unlock(&iaa_devices_lock);
2084	mutex_unlock(&wq->wq_lock);
2085}
2086
2087static enum idxd_dev_type dev_types[] = {
2088	IDXD_DEV_WQ,
2089	IDXD_DEV_NONE,
2090};
2091
2092static struct idxd_device_driver iaa_crypto_driver = {
2093	.probe = iaa_crypto_probe,
2094	.remove = iaa_crypto_remove,
2095	.name = IDXD_SUBDRIVER_NAME,
2096	.type = dev_types,
2097	.desc_complete = iaa_desc_complete,
2098};
2099
2100static int __init iaa_crypto_init_module(void)
2101{
2102	int ret = 0;
2103	int node;
2104
2105	nr_cpus = num_online_cpus();
2106	for_each_node_with_cpus(node)
2107		nr_nodes++;
2108	if (!nr_nodes) {
2109		pr_err("IAA couldn't find any nodes with cpus\n");
2110		return -ENODEV;
2111	}
2112	nr_cpus_per_node = nr_cpus / nr_nodes;
2113
2114	if (crypto_has_comp("deflate-generic", 0, 0))
2115		deflate_generic_tfm = crypto_alloc_comp("deflate-generic", 0, 0);
2116
2117	if (IS_ERR_OR_NULL(deflate_generic_tfm)) {
2118		pr_err("IAA could not alloc %s tfm: errcode = %ld\n",
2119		       "deflate-generic", PTR_ERR(deflate_generic_tfm));
2120		return -ENOMEM;
2121	}
2122
2123	ret = iaa_aecs_init_fixed();
2124	if (ret < 0) {
2125		pr_debug("IAA fixed compression mode init failed\n");
2126		goto err_aecs_init;
2127	}
2128
2129	ret = idxd_driver_register(&iaa_crypto_driver);
2130	if (ret) {
2131		pr_debug("IAA wq sub-driver registration failed\n");
2132		goto err_driver_reg;
2133	}
2134
2135	ret = driver_create_file(&iaa_crypto_driver.drv,
2136				 &driver_attr_verify_compress);
2137	if (ret) {
2138		pr_debug("IAA verify_compress attr creation failed\n");
2139		goto err_verify_attr_create;
2140	}
2141
2142	ret = driver_create_file(&iaa_crypto_driver.drv,
2143				 &driver_attr_sync_mode);
2144	if (ret) {
2145		pr_debug("IAA sync mode attr creation failed\n");
2146		goto err_sync_attr_create;
2147	}
2148
2149	if (iaa_crypto_debugfs_init())
2150		pr_warn("debugfs init failed, stats not available\n");
2151
2152	pr_debug("initialized\n");
2153out:
2154	return ret;
2155
2156err_sync_attr_create:
2157	driver_remove_file(&iaa_crypto_driver.drv,
2158			   &driver_attr_verify_compress);
2159err_verify_attr_create:
2160	idxd_driver_unregister(&iaa_crypto_driver);
2161err_driver_reg:
2162	iaa_aecs_cleanup_fixed();
2163err_aecs_init:
2164	crypto_free_comp(deflate_generic_tfm);
2165
2166	goto out;
2167}
2168
2169static void __exit iaa_crypto_cleanup_module(void)
2170{
2171	if (iaa_unregister_compression_device())
2172		pr_debug("IAA compression device unregister failed\n");
2173
2174	iaa_crypto_debugfs_cleanup();
2175	driver_remove_file(&iaa_crypto_driver.drv,
2176			   &driver_attr_sync_mode);
2177	driver_remove_file(&iaa_crypto_driver.drv,
2178			   &driver_attr_verify_compress);
2179	idxd_driver_unregister(&iaa_crypto_driver);
2180	iaa_aecs_cleanup_fixed();
2181	crypto_free_comp(deflate_generic_tfm);
2182
2183	pr_debug("cleaned up\n");
2184}
2185
2186MODULE_IMPORT_NS(IDXD);
2187MODULE_LICENSE("GPL");
2188MODULE_ALIAS_IDXD_DEVICE(0);
2189MODULE_AUTHOR("Intel Corporation");
2190MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver");
2191
2192module_init(iaa_crypto_init_module);
2193module_exit(iaa_crypto_cleanup_module);