From nobody Thu Oct 2 02:13:25 2025 Received: from mgamail.intel.com (mgamail.intel.com [192.198.163.13]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 72E881D63C2; Fri, 26 Sep 2025 03:35:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=192.198.163.13 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758857709; cv=none; b=UbnMKYbOxObY9ABFOZHrT4Ud72twJ8AeYMFaCWzGLbWVaUP+MmYW1MfxkNzkfEjfU8Oqzm4rLw47iBXxfoVeJSSmHw93cwicyl8OsCJ21NE1xgqdxrGbkDx1rHS6usNf6jJ6s7FvquwiuJUTs/WXd9GC2DVJD5B74UVW2o5G22Y= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758857709; c=relaxed/simple; bh=pev2rWbFSDQItiAHUxDqBId8SFZZt9oAc87aO1i4JWM=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=cDE/1HVbv/7u+IECh4DFWTwS0LW7D1A52soqKE1AON61mJJUxB24u7W9U7WA14Uo/3fXf7VpBoGCjkpAOPJOAA24F8CTpIWV9VhOV7g7eXscHYGpDrRalrlVk0vXSMLHwpskEnwFt2Ex2ZcPZ+oDVodD3JSLa+cDX91rbzKOyaI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=intel.com; spf=pass smtp.mailfrom=intel.com; dkim=pass (2048-bit key) header.d=intel.com header.i=@intel.com header.b=k9Db4b8P; arc=none smtp.client-ip=192.198.163.13 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=intel.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=intel.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=intel.com header.i=@intel.com header.b="k9Db4b8P" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1758857707; x=1790393707; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=pev2rWbFSDQItiAHUxDqBId8SFZZt9oAc87aO1i4JWM=; b=k9Db4b8PTNfQmBoTQtScIABongeM+tiPrb/DwxSzqEJa2gF5aO6llGRB 2QvVwK7TSI0Ph92GItPrtbiLpoO4u7ALcmXJzEZVWp/dD5VWrSw3hsIoL T/Lg37uoBxBJj3qjlv7zakTzIWCSRAeuqaD0IMNAf8CnMpa8S9CbFS+fI jSqFwIjRbZbWfa8PoYYekalESrm1odqf+GOyLmt7I2QEZ7Kt6+7fwZCDM 2yN1leGZ+5sMriOfg4FBEQ8Uy3Up9uiN4o3AD59JY7815/R+WMVdKswEM S6ks6B1xkcd6YI114ZcC4g+8s8F5f5ryWQ5f8R0wtMGpFNu4Mnr648WZv Q==; X-CSE-ConnectionGUID: 4wvqXRvjQ4umrdegmOfJOw== X-CSE-MsgGUID: Aq59FD7HSnq7aqf8MGwNvQ== X-IronPort-AV: E=McAfee;i="6800,10657,11564"; a="63819403" X-IronPort-AV: E=Sophos;i="6.18,294,1751266800"; d="scan'208";a="63819403" Received: from orviesa001.jf.intel.com ([10.64.159.141]) by fmvoesa107.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 25 Sep 2025 20:35:04 -0700 X-CSE-ConnectionGUID: 1T5PToXfQ1a7NhfCz7mXig== X-CSE-MsgGUID: Tu1BiAufQF+RvwZRNZ4/eg== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="6.18,294,1751266800"; d="scan'208";a="214636546" Received: from jf5300-b11a338t.jf.intel.com ([10.242.51.115]) by orviesa001.jf.intel.com with ESMTP; 25 Sep 2025 20:35:02 -0700 From: Kanchana P Sridhar To: linux-kernel@vger.kernel.org, linux-mm@kvack.org, hannes@cmpxchg.org, yosry.ahmed@linux.dev, nphamcs@gmail.com, chengming.zhou@linux.dev, usamaarif642@gmail.com, ryan.roberts@arm.com, 21cnbao@gmail.com, ying.huang@linux.alibaba.com, akpm@linux-foundation.org, senozhatsky@chromium.org, sj@kernel.org, kasong@tencent.com, linux-crypto@vger.kernel.org, herbert@gondor.apana.org.au, davem@davemloft.net, clabbe@baylibre.com, ardb@kernel.org, ebiggers@google.com, surenb@google.com, kristen.c.accardi@intel.com, vinicius.gomes@intel.com Cc: wajdi.k.feghali@intel.com, vinodh.gopal@intel.com, kanchana.p.sridhar@intel.com Subject: [PATCH v12 01/23] crypto: iaa - Reorganize the iaa_crypto driver code. Date: Thu, 25 Sep 2025 20:34:40 -0700 Message-Id: <20250926033502.7486-2-kanchana.p.sridhar@intel.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20250926033502.7486-1-kanchana.p.sridhar@intel.com> References: <20250926033502.7486-1-kanchana.p.sridhar@intel.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This patch merely reorganizes the code in iaa_crypto_main.c, so that the functions are consolidated into logically related sub-sections of code, without requiring forward declarations. This is expected to make the code more maintainable and for it to be easier to replace functional layers and/or add new features. Signed-off-by: Kanchana P Sridhar --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 677 +++++++++++---------- 1 file changed, 350 insertions(+), 327 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/in= tel/iaa/iaa_crypto_main.c index 23f585219fb4..760997eee8fe 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -24,6 +24,10 @@ =20 #define IAA_ALG_PRIORITY 300 =20 +/************************************** + * Driver internal global variables. + **************************************/ + /* number of iaa instances probed */ static unsigned int nr_iaa; static unsigned int nr_cpus; @@ -36,54 +40,6 @@ static unsigned int cpus_per_iaa; /* Per-cpu lookup table for balanced wqs */ static struct wq_table_entry __percpu *wq_table; =20 -static struct idxd_wq *wq_table_next_wq(int cpu) -{ - struct wq_table_entry *entry =3D per_cpu_ptr(wq_table, cpu); - - if (++entry->cur_wq >=3D entry->n_wqs) - entry->cur_wq =3D 0; - - if (!entry->wqs[entry->cur_wq]) - return NULL; - - pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __fun= c__, - entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id, - entry->wqs[entry->cur_wq]->id, cpu); - - return entry->wqs[entry->cur_wq]; -} - -static void wq_table_add(int cpu, struct idxd_wq *wq) -{ - struct wq_table_entry *entry =3D per_cpu_ptr(wq_table, cpu); - - if (WARN_ON(entry->n_wqs =3D=3D entry->max_wqs)) - return; - - entry->wqs[entry->n_wqs++] =3D wq; - - pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__, - entry->wqs[entry->n_wqs - 1]->idxd->id, - entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu); -} - -static void wq_table_free_entry(int cpu) -{ - struct wq_table_entry *entry =3D per_cpu_ptr(wq_table, cpu); - - kfree(entry->wqs); - memset(entry, 0, sizeof(*entry)); -} - -static void wq_table_clear_entry(int cpu) -{ - struct wq_table_entry *entry =3D per_cpu_ptr(wq_table, cpu); - - entry->n_wqs =3D 0; - entry->cur_wq =3D 0; - memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *)); -} - LIST_HEAD(iaa_devices); DEFINE_MUTEX(iaa_devices_lock); =20 @@ -91,36 +47,11 @@ DEFINE_MUTEX(iaa_devices_lock); static bool iaa_crypto_enabled; static bool iaa_crypto_registered; =20 +static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_M= AX]; + /* Verify results of IAA compress or not */ static bool iaa_verify_compress =3D true; =20 -static ssize_t verify_compress_show(struct device_driver *driver, char *bu= f) -{ - return sprintf(buf, "%d\n", iaa_verify_compress); -} - -static ssize_t verify_compress_store(struct device_driver *driver, - const char *buf, size_t count) -{ - int ret =3D -EBUSY; - - mutex_lock(&iaa_devices_lock); - - if (iaa_crypto_enabled) - goto out; - - ret =3D kstrtobool(buf, &iaa_verify_compress); - if (ret) - goto out; - - ret =3D count; -out: - mutex_unlock(&iaa_devices_lock); - - return ret; -} -static DRIVER_ATTR_RW(verify_compress); - /* * The iaa crypto driver supports three 'sync' methods determining how * compressions and decompressions are performed: @@ -155,6 +86,37 @@ static bool async_mode; /* Use interrupts */ static bool use_irq; =20 +/************************************************** + * Driver attributes along with get/set functions. + **************************************************/ + +static ssize_t verify_compress_show(struct device_driver *driver, char *bu= f) +{ + return sprintf(buf, "%d\n", iaa_verify_compress); +} + +static ssize_t verify_compress_store(struct device_driver *driver, + const char *buf, size_t count) +{ + int ret =3D -EBUSY; + + mutex_lock(&iaa_devices_lock); + + if (iaa_crypto_enabled) + goto out; + + ret =3D kstrtobool(buf, &iaa_verify_compress); + if (ret) + goto out; + + ret =3D count; +out: + mutex_unlock(&iaa_devices_lock); + + return ret; +} +static DRIVER_ATTR_RW(verify_compress); + /** * set_iaa_sync_mode - Set IAA sync mode * @name: The name of the sync mode @@ -217,7 +179,9 @@ static ssize_t sync_mode_store(struct device_driver *dr= iver, } static DRIVER_ATTR_RW(sync_mode); =20 -static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_M= AX]; +/**************************** + * Driver compression modes. + ****************************/ =20 static int find_empty_iaa_compression_mode(void) { @@ -409,11 +373,6 @@ static void free_device_compression_mode(struct iaa_de= vice *iaa_device, IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \ IDXD_OP_FLAG_AECS_RW_TGLS) =20 -static int check_completion(struct device *dev, - struct iax_completion_record *comp, - bool compress, - bool only_once); - static int init_device_compression_mode(struct iaa_device *iaa_device, struct iaa_compression_mode *mode, int idx, struct idxd_wq *wq) @@ -500,6 +459,11 @@ static void remove_device_compression_modes(struct iaa= _device *iaa_device) } } =20 +/*********************************************************** + * Functions for use in crypto probe and remove interfaces: + * allocate/init/query/deallocate devices/wqs. + ***********************************************************/ + static struct iaa_device *iaa_device_alloc(void) { struct iaa_device *iaa_device; @@ -513,18 +477,6 @@ static struct iaa_device *iaa_device_alloc(void) return iaa_device; } =20 -static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq) -{ - struct iaa_wq *iaa_wq; - - list_for_each_entry(iaa_wq, &iaa_device->wqs, list) { - if (iaa_wq->wq =3D=3D wq) - return true; - } - - return false; -} - static struct iaa_device *add_iaa_device(struct idxd_device *idxd) { struct iaa_device *iaa_device; @@ -560,6 +512,27 @@ static void del_iaa_device(struct iaa_device *iaa_devi= ce) nr_iaa--; } =20 +static void free_iaa_device(struct iaa_device *iaa_device) +{ + if (!iaa_device) + return; + + remove_device_compression_modes(iaa_device); + kfree(iaa_device); +} + +static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq) +{ + struct iaa_wq *iaa_wq; + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) { + if (iaa_wq->wq =3D=3D wq) + return true; + } + + return false; +} + static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq, struct iaa_wq **new_wq) { @@ -612,23 +585,23 @@ static void del_iaa_wq(struct iaa_device *iaa_device,= struct idxd_wq *wq) } } =20 -static void clear_wq_table(void) +static void remove_iaa_wq(struct idxd_wq *wq) { - int cpu; - - for (cpu =3D 0; cpu < nr_cpus; cpu++) - wq_table_clear_entry(cpu); - - pr_debug("cleared wq table\n"); -} + struct iaa_device *iaa_device; =20 -static void free_iaa_device(struct iaa_device *iaa_device) -{ - if (!iaa_device) - return; + list_for_each_entry(iaa_device, &iaa_devices, list) { + if (iaa_has_wq(iaa_device, wq)) { + del_iaa_wq(iaa_device, wq); + break; + } + } =20 - remove_device_compression_modes(iaa_device); - kfree(iaa_device); + if (nr_iaa) { + cpus_per_iaa =3D (nr_nodes * nr_cpus_per_node) / nr_iaa; + if (!cpus_per_iaa) + cpus_per_iaa =3D 1; + } else + cpus_per_iaa =3D 1; } =20 static void __free_iaa_wq(struct iaa_wq *iaa_wq) @@ -655,6 +628,75 @@ static void free_iaa_wq(struct iaa_wq *iaa_wq) idxd_wq_set_private(wq, NULL); } =20 +static int save_iaa_wq(struct idxd_wq *wq) +{ + struct iaa_device *iaa_device, *found =3D NULL; + struct idxd_device *idxd; + struct pci_dev *pdev; + struct device *dev; + int ret =3D 0; + + list_for_each_entry(iaa_device, &iaa_devices, list) { + if (iaa_device->idxd =3D=3D wq->idxd) { + idxd =3D iaa_device->idxd; + pdev =3D idxd->pdev; + dev =3D &pdev->dev; + /* + * Check to see that we don't already have this wq. + * Shouldn't happen but we don't control probing. + */ + if (iaa_has_wq(iaa_device, wq)) { + dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n", + iaa_device); + goto out; + } + + found =3D iaa_device; + + ret =3D add_iaa_wq(iaa_device, wq, NULL); + if (ret) + goto out; + + break; + } + } + + if (!found) { + struct iaa_device *new_device; + struct iaa_wq *new_wq; + + new_device =3D add_iaa_device(wq->idxd); + if (!new_device) { + ret =3D -ENOMEM; + goto out; + } + + ret =3D add_iaa_wq(new_device, wq, &new_wq); + if (ret) { + del_iaa_device(new_device); + free_iaa_device(new_device); + goto out; + } + + ret =3D init_iaa_device(new_device, new_wq); + if (ret) { + del_iaa_wq(new_device, new_wq->wq); + del_iaa_device(new_device); + free_iaa_wq(new_wq); + goto out; + } + } + + if (WARN_ON(nr_iaa =3D=3D 0)) + return -EINVAL; + + cpus_per_iaa =3D (nr_nodes * nr_cpus_per_node) / nr_iaa; + if (!cpus_per_iaa) + cpus_per_iaa =3D 1; +out: + return 0; +} + static int iaa_wq_get(struct idxd_wq *wq) { struct idxd_device *idxd =3D wq->idxd; @@ -702,6 +744,37 @@ static int iaa_wq_put(struct idxd_wq *wq) return ret; } =20 +/*************************************************************** + * Mapping IAA devices and wqs to cores with per-cpu wq_tables. + ***************************************************************/ + +static void wq_table_free_entry(int cpu) +{ + struct wq_table_entry *entry =3D per_cpu_ptr(wq_table, cpu); + + kfree(entry->wqs); + memset(entry, 0, sizeof(*entry)); +} + +static void wq_table_clear_entry(int cpu) +{ + struct wq_table_entry *entry =3D per_cpu_ptr(wq_table, cpu); + + entry->n_wqs =3D 0; + entry->cur_wq =3D 0; + memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *)); +} + +static void clear_wq_table(void) +{ + int cpu; + + for (cpu =3D 0; cpu < nr_cpus; cpu++) + wq_table_clear_entry(cpu); + + pr_debug("cleared wq table\n"); +} + static void free_wq_table(void) { int cpu; @@ -739,92 +812,18 @@ static int alloc_wq_table(int max_wqs) return 0; } =20 -static int save_iaa_wq(struct idxd_wq *wq) +static void wq_table_add(int cpu, struct idxd_wq *wq) { - struct iaa_device *iaa_device, *found =3D NULL; - struct idxd_device *idxd; - struct pci_dev *pdev; - struct device *dev; - int ret =3D 0; - - list_for_each_entry(iaa_device, &iaa_devices, list) { - if (iaa_device->idxd =3D=3D wq->idxd) { - idxd =3D iaa_device->idxd; - pdev =3D idxd->pdev; - dev =3D &pdev->dev; - /* - * Check to see that we don't already have this wq. - * Shouldn't happen but we don't control probing. - */ - if (iaa_has_wq(iaa_device, wq)) { - dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n", - iaa_device); - goto out; - } - - found =3D iaa_device; - - ret =3D add_iaa_wq(iaa_device, wq, NULL); - if (ret) - goto out; - - break; - } - } - - if (!found) { - struct iaa_device *new_device; - struct iaa_wq *new_wq; - - new_device =3D add_iaa_device(wq->idxd); - if (!new_device) { - ret =3D -ENOMEM; - goto out; - } - - ret =3D add_iaa_wq(new_device, wq, &new_wq); - if (ret) { - del_iaa_device(new_device); - free_iaa_device(new_device); - goto out; - } - - ret =3D init_iaa_device(new_device, new_wq); - if (ret) { - del_iaa_wq(new_device, new_wq->wq); - del_iaa_device(new_device); - free_iaa_wq(new_wq); - goto out; - } - } - - if (WARN_ON(nr_iaa =3D=3D 0)) - return -EINVAL; - - cpus_per_iaa =3D (nr_nodes * nr_cpus_per_node) / nr_iaa; - if (!cpus_per_iaa) - cpus_per_iaa =3D 1; -out: - return 0; -} + struct wq_table_entry *entry =3D per_cpu_ptr(wq_table, cpu); =20 -static void remove_iaa_wq(struct idxd_wq *wq) -{ - struct iaa_device *iaa_device; + if (WARN_ON(entry->n_wqs =3D=3D entry->max_wqs)) + return; =20 - list_for_each_entry(iaa_device, &iaa_devices, list) { - if (iaa_has_wq(iaa_device, wq)) { - del_iaa_wq(iaa_device, wq); - break; - } - } + entry->wqs[entry->n_wqs++] =3D wq; =20 - if (nr_iaa) { - cpus_per_iaa =3D (nr_nodes * nr_cpus_per_node) / nr_iaa; - if (!cpus_per_iaa) - cpus_per_iaa =3D 1; - } else - cpus_per_iaa =3D 1; + pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__, + entry->wqs[entry->n_wqs - 1]->idxd->id, + entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu); } =20 static int wq_table_add_wqs(int iaa, int cpu) @@ -930,6 +929,44 @@ static void rebalance_wq_table(void) pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); } =20 +/*************************************************************** + * Assign work-queues for driver ops using per-cpu wq_tables. + ***************************************************************/ + +static struct idxd_wq *wq_table_next_wq(int cpu) +{ + struct wq_table_entry *entry =3D per_cpu_ptr(wq_table, cpu); + + if (++entry->cur_wq >=3D entry->n_wqs) + entry->cur_wq =3D 0; + + if (!entry->wqs[entry->cur_wq]) + return NULL; + + pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __fun= c__, + entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id, + entry->wqs[entry->cur_wq]->id, cpu); + + return entry->wqs[entry->cur_wq]; +} + +/************************************************* + * Core iaa_crypto compress/decompress functions. + *************************************************/ + +static int deflate_generic_decompress(struct acomp_req *req) +{ + ACOMP_FBREQ_ON_STACK(fbreq, req); + int ret; + + ret =3D crypto_acomp_decompress(fbreq); + req->dlen =3D fbreq->dlen; + + update_total_sw_decomp_calls(); + + return ret; +} + static inline int check_completion(struct device *dev, struct iax_completion_record *comp, bool compress, @@ -990,27 +1027,132 @@ static inline int check_completion(struct device *d= ev, return ret; } =20 -static int deflate_generic_decompress(struct acomp_req *req) +static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, + struct acomp_req *req, + dma_addr_t *src_addr, dma_addr_t *dst_addr) { - ACOMP_FBREQ_ON_STACK(fbreq, req); - int ret; + int ret =3D 0; + int nr_sgs; =20 - ret =3D crypto_acomp_decompress(fbreq); - req->dlen =3D fbreq->dlen; + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); =20 - update_total_sw_decomp_calls(); + nr_sgs =3D dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + if (nr_sgs <=3D 0 || nr_sgs > 1) { + dev_dbg(dev, "verify: couldn't map src sg for iaa device %d," + " wq %d: ret=3D%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret =3D -EIO; + goto out; + } + *src_addr =3D sg_dma_address(req->src); + dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," + " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs, + req->src, req->slen, sg_dma_len(req->src)); =20 + nr_sgs =3D dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); + if (nr_sgs <=3D 0 || nr_sgs > 1) { + dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d," + " wq %d: ret=3D%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret =3D -EIO; + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + goto out; + } + *dst_addr =3D sg_dma_address(req->dst); + dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," + " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs, + req->dst, req->dlen, sg_dma_len(req->dst)); +out: return ret; } =20 -static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, - struct acomp_req *req, - dma_addr_t *src_addr, dma_addr_t *dst_addr); - static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *r= eq, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen); + dma_addr_t dst_addr, unsigned int *dlen) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *ctx =3D crypto_tfm_ctx(tfm); + u32 *compression_crc =3D acomp_request_ctx(req); + struct iaa_device *iaa_device; + struct idxd_desc *idxd_desc; + struct iax_hw_desc *desc; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret =3D 0; + + iaa_wq =3D idxd_wq_get_private(wq); + iaa_device =3D iaa_wq->iaa_device; + idxd =3D iaa_device->idxd; + pdev =3D idxd->pdev; + dev =3D &pdev->dev; + + active_compression_mode =3D get_iaa_device_compression_mode(iaa_device, c= tx->mode); + + idxd_desc =3D idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(idxd_desc)) { + dev_dbg(dev, "idxd descriptor allocation failed\n"); + dev_dbg(dev, "iaa compress failed: ret=3D%ld\n", + PTR_ERR(idxd_desc)); + return PTR_ERR(idxd_desc); + } + desc =3D idxd_desc->iax_hw; + + /* Verify (optional) - decompress and check crc, suppress dest write */ + + desc->flags =3D IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC; + desc->opcode =3D IAX_OPCODE_DECOMPRESS; + desc->decompr_flags =3D IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT; + desc->priv =3D 0; + + desc->src1_addr =3D (u64)dst_addr; + desc->src1_size =3D *dlen; + desc->dst_addr =3D (u64)src_addr; + desc->max_dst_size =3D slen; + desc->completion_addr =3D idxd_desc->compl_dma; + + dev_dbg(dev, "(verify) compression mode %s," + " desc->src1_addr %llx, desc->src1_size %d," + " desc->dst_addr %llx, desc->max_dst_size %d," + " desc->src2_addr %llx, desc->src2_size %d\n", + active_compression_mode->name, + desc->src1_addr, desc->src1_size, desc->dst_addr, + desc->max_dst_size, desc->src2_addr, desc->src2_size); + + ret =3D idxd_submit_desc(wq, idxd_desc); + if (ret) { + dev_dbg(dev, "submit_desc (verify) failed ret=3D%d\n", ret); + goto err; + } + + ret =3D check_completion(dev, idxd_desc->iax_completion, false, false); + if (ret) { + dev_dbg(dev, "(verify) check_completion failed ret=3D%d\n", ret); + goto err; + } + + if (*compression_crc !=3D idxd_desc->iax_completion->crc) { + ret =3D -EINVAL; + dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" + " comp=3D0x%x, decomp=3D0x%x\n", *compression_crc, + idxd_desc->iax_completion->crc); + print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, + 8, 1, idxd_desc->iax_completion, 64, 0); + goto err; + } + + idxd_free_desc(wq, idxd_desc); +out: + return ret; +err: + idxd_free_desc(wq, idxd_desc); + dev_dbg(dev, "iaa compress failed: ret=3D%d\n", ret); + + goto out; +} =20 static void iaa_desc_complete(struct idxd_desc *idxd_desc, enum idxd_complete_type comp_type, @@ -1226,133 +1368,6 @@ static int iaa_compress(struct crypto_tfm *tfm, str= uct acomp_req *req, goto out; } =20 -static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, - struct acomp_req *req, - dma_addr_t *src_addr, dma_addr_t *dst_addr) -{ - int ret =3D 0; - int nr_sgs; - - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - - nr_sgs =3D dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); - if (nr_sgs <=3D 0 || nr_sgs > 1) { - dev_dbg(dev, "verify: couldn't map src sg for iaa device %d," - " wq %d: ret=3D%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret =3D -EIO; - goto out; - } - *src_addr =3D sg_dma_address(req->src); - dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs, - req->src, req->slen, sg_dma_len(req->src)); - - nr_sgs =3D dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); - if (nr_sgs <=3D 0 || nr_sgs > 1) { - dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d," - " wq %d: ret=3D%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret =3D -EIO; - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); - goto out; - } - *dst_addr =3D sg_dma_address(req->dst); - dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs, - req->dst, req->dlen, sg_dma_len(req->dst)); -out: - return ret; -} - -static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *r= eq, - struct idxd_wq *wq, - dma_addr_t src_addr, unsigned int slen, - dma_addr_t dst_addr, unsigned int *dlen) -{ - struct iaa_device_compression_mode *active_compression_mode; - struct iaa_compression_ctx *ctx =3D crypto_tfm_ctx(tfm); - u32 *compression_crc =3D acomp_request_ctx(req); - struct iaa_device *iaa_device; - struct idxd_desc *idxd_desc; - struct iax_hw_desc *desc; - struct idxd_device *idxd; - struct iaa_wq *iaa_wq; - struct pci_dev *pdev; - struct device *dev; - int ret =3D 0; - - iaa_wq =3D idxd_wq_get_private(wq); - iaa_device =3D iaa_wq->iaa_device; - idxd =3D iaa_device->idxd; - pdev =3D idxd->pdev; - dev =3D &pdev->dev; - - active_compression_mode =3D get_iaa_device_compression_mode(iaa_device, c= tx->mode); - - idxd_desc =3D idxd_alloc_desc(wq, IDXD_OP_BLOCK); - if (IS_ERR(idxd_desc)) { - dev_dbg(dev, "idxd descriptor allocation failed\n"); - dev_dbg(dev, "iaa compress failed: ret=3D%ld\n", - PTR_ERR(idxd_desc)); - return PTR_ERR(idxd_desc); - } - desc =3D idxd_desc->iax_hw; - - /* Verify (optional) - decompress and check crc, suppress dest write */ - - desc->flags =3D IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC; - desc->opcode =3D IAX_OPCODE_DECOMPRESS; - desc->decompr_flags =3D IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT; - desc->priv =3D 0; - - desc->src1_addr =3D (u64)dst_addr; - desc->src1_size =3D *dlen; - desc->dst_addr =3D (u64)src_addr; - desc->max_dst_size =3D slen; - desc->completion_addr =3D idxd_desc->compl_dma; - - dev_dbg(dev, "(verify) compression mode %s," - " desc->src1_addr %llx, desc->src1_size %d," - " desc->dst_addr %llx, desc->max_dst_size %d," - " desc->src2_addr %llx, desc->src2_size %d\n", - active_compression_mode->name, - desc->src1_addr, desc->src1_size, desc->dst_addr, - desc->max_dst_size, desc->src2_addr, desc->src2_size); - - ret =3D idxd_submit_desc(wq, idxd_desc); - if (ret) { - dev_dbg(dev, "submit_desc (verify) failed ret=3D%d\n", ret); - goto err; - } - - ret =3D check_completion(dev, idxd_desc->iax_completion, false, false); - if (ret) { - dev_dbg(dev, "(verify) check_completion failed ret=3D%d\n", ret); - goto err; - } - - if (*compression_crc !=3D idxd_desc->iax_completion->crc) { - ret =3D -EINVAL; - dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" - " comp=3D0x%x, decomp=3D0x%x\n", *compression_crc, - idxd_desc->iax_completion->crc); - print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, - 8, 1, idxd_desc->iax_completion, 64, 0); - goto err; - } - - idxd_free_desc(wq, idxd_desc); -out: - return ret; -err: - idxd_free_desc(wq, idxd_desc); - dev_dbg(dev, "iaa compress failed: ret=3D%d\n", ret); - - goto out; -} - static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, struct idxd_wq *wq, dma_addr_t src_addr, unsigned int slen, @@ -1662,6 +1677,10 @@ static void compression_ctx_init(struct iaa_compress= ion_ctx *ctx) ctx->use_irq =3D use_irq; } =20 +/********************************************* + * Interfaces to crypto_alg and crypto_acomp. + *********************************************/ + static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm) { struct crypto_tfm *tfm =3D crypto_acomp_tfm(acomp_tfm); @@ -1864,6 +1883,10 @@ static struct idxd_device_driver iaa_crypto_driver = =3D { .desc_complete =3D iaa_desc_complete, }; =20 +/******************** + * Module init/exit. + ********************/ + static int __init iaa_crypto_init_module(void) { int ret =3D 0; --=20 2.27.0