fs/erofs/Kconfig | 14 +++ fs/erofs/Makefile | 1 + fs/erofs/compress.h | 14 +++ fs/erofs/decompressor_crypto.c | 199 ++++++++++++++++++++++++++++++++ fs/erofs/decompressor_deflate.c | 16 ++- fs/erofs/sysfs.c | 36 ++++++ 6 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 fs/erofs/decompressor_crypto.c
This patch introdueces the use of the Intel QAT to decompress compressed
data in the EROFS filesystem, aiming to improve the decompression speed
of compressed datea.
We created a 285MiB compressed file and then used the following command to
create EROFS images with different cluster size.
# mkfs.erofs -zdeflate,level=9 -C16384
fio command was used to test random read and small random read(~5%) and
sequential read performance.
# fio -filename=testfile -bs=4k -rw=read -name=job1
# fio -filename=testfile -bs=4k -rw=randread -name=job1
# fio -filename=testfile -bs=4k -rw=randread --io_size=14m -name=job1
Here are some performance numbers for reference:
Processors: Intel(R) Xeon(R) 6766E(144 core)
Memory: 521 GiB
|-----------------------------------------------------------------------------|
| | Cluster size | sequential read | randread | small randread(5%) |
|-----------|--------------|-----------------|-----------|--------------------|
| Intel QAT | 4096 | 538 MiB/s | 112 MiB/s | 20.76 MiB/s |
| Intel QAT | 16384 | 699 MiB/s | 158 MiB/s | 21.02 MiB/s |
| Intel QAT | 65536 | 917 MiB/s | 278 MiB/s | 20.90 MiB/s |
| Intel QAT | 131072 | 1056 MiB/s | 351 MiB/s | 23.36 MiB/s |
| Intel QAT | 262144 | 1145 MiB/s | 431 MiB/s | 26.66 MiB/s |
| deflate | 4096 | 499 MiB/s | 108 MiB/s | 21.50 MiB/s |
| deflate | 16384 | 422 MiB/s | 125 MiB/s | 18.94 MiB/s |
| deflate | 65536 | 452 MiB/s | 159 MiB/s | 13.02 MiB/s |
| deflate | 131072 | 452 MiB/s | 177 MiB/s | 11.44 MiB/s |
| deflate | 262144 | 466 MiB/s | 194 MiB/s | 10.60 MiB/s |
Signed-off-by: Bo Liu <liubo03@inspur.com>
---
v1: https://lore.kernel.org/linux-erofs/20250410042048.3044-1-liubo03@inspur.com/
v2: https://lore.kernel.org/linux-erofs/20250410042048.3044-1-liubo03@inspur.com/T/#t
Changes since v2:
- Add a new kernel config option CONFIG_EROFS_FS_ZIP_CRYPTO.
- Modify the sysfs interface to merge enable and disable into
a single entry point, and add a read function to view the current state.
- Optimize the relevant code,use the sg_alloc_table_from_pages_segment interface.
fs/erofs/Kconfig | 14 +++
fs/erofs/Makefile | 1 +
fs/erofs/compress.h | 14 +++
fs/erofs/decompressor_crypto.c | 199 ++++++++++++++++++++++++++++++++
fs/erofs/decompressor_deflate.c | 16 ++-
fs/erofs/sysfs.c | 36 ++++++
6 files changed, 279 insertions(+), 1 deletion(-)
create mode 100644 fs/erofs/decompressor_crypto.c
diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig
index 331e49cd1b8d..db49ae3c1922 100644
--- a/fs/erofs/Kconfig
+++ b/fs/erofs/Kconfig
@@ -144,6 +144,20 @@ config EROFS_FS_ZIP_ZSTD
If unsure, say N.
+config EROFS_FS_ZIP_CRYPTO
+ bool "EROFS hardware decompression support (crypto interface)"
+ depends on EROFS_FS_ZIP
+ help
+ Saying Y here includes support for reading EROFS file systems
+ containing crypto compressed data. It gives better decompression
+ speed than the software-implemented compression, and it costs
+ lower CPU overhead.
+
+ Crypto support is an experimental feature for now and so most
+ file systems will be readable without selecting this option.
+
+ If unsure, say N.
+
config EROFS_FS_ONDEMAND
bool "EROFS fscache-based on-demand read support (deprecated)"
depends on EROFS_FS
diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile
index 4331d53c7109..247b263eb667 100644
--- a/fs/erofs/Makefile
+++ b/fs/erofs/Makefile
@@ -7,5 +7,6 @@ erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o zutil.o
erofs-$(CONFIG_EROFS_FS_ZIP_LZMA) += decompressor_lzma.o
erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o
erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o
+erofs-$(CONFIG_EROFS_FS_ZIP_CRYPTO) += decompressor_crypto.o
erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o
erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o
diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h
index 2704d7a592a5..356ae2473bd5 100644
--- a/fs/erofs/compress.h
+++ b/fs/erofs/compress.h
@@ -70,10 +70,24 @@ struct z_erofs_stream_dctx {
bool bounced; /* is the bounce buffer used now? */
};
+struct z_erofs_crypto_engine {
+ char *crypto_name;
+ bool enabled;
+ struct crypto_acomp *erofs_tfm;
+};
+
+extern struct z_erofs_crypto_engine z_erofs_crypto[Z_EROFS_COMPRESSION_MAX][2];
+
int z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, void **dst,
void **src, struct page **pgpl);
int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,
unsigned int padbufsize);
int __init z_erofs_init_decompressor(void);
void z_erofs_exit_decompressor(void);
+int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
+ struct crypto_acomp *erofs_tfm, struct page **pgpl);
+struct crypto_acomp *z_erofs_crypto_get_engine(int type);
+int z_erofs_crypto_enable_engine(const char *name);
+void z_erofs_crypto_disable_engine(void);
+int z_erofs_crypto_engine_format(char *buf);
#endif
diff --git a/fs/erofs/decompressor_crypto.c b/fs/erofs/decompressor_crypto.c
new file mode 100644
index 000000000000..55f6a9e6dcf6
--- /dev/null
+++ b/fs/erofs/decompressor_crypto.c
@@ -0,0 +1,199 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/scatterlist.h>
+#include <crypto/acompress.h>
+
+#include "compress.h"
+
+static int z_erofs_crypto_decompress_mem(struct z_erofs_decompress_req *rq,
+ struct crypto_acomp *erofs_tfm)
+{
+ struct sg_table st_src, st_dst;
+ struct acomp_req *req;
+ struct crypto_wait wait;
+ u8 *headpage;
+ int ret;
+
+ headpage = kmap_local_page(*rq->in);
+ ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
+ min_t(unsigned int, rq->inputsize,
+ rq->sb->s_blocksize - rq->pageofs_in));
+ kunmap_local(headpage);
+ if (ret)
+ return ret;
+
+ req = acomp_request_alloc(erofs_tfm);
+ if (!req) {
+ erofs_err(rq->sb, "failed to alloc decompress request");
+ return -ENOMEM;
+ }
+
+ ret = sg_alloc_table_from_pages_segment(&st_src,
+ rq->in, rq->inpages, rq->pageofs_in,
+ rq->inputsize, UINT_MAX, GFP_KERNEL);
+ if (ret < 0)
+ goto failed_src_alloc;
+
+ ret = sg_alloc_table_from_pages_segment(&st_dst,
+ rq->out, rq->outpages, rq->pageofs_out,
+ rq->outputsize, UINT_MAX, GFP_KERNEL);
+ if (ret < 0)
+ goto failed_dst_alloc;
+
+ acomp_request_set_params(req, st_src.sgl,
+ st_dst.sgl, rq->inputsize, rq->outputsize);
+
+ crypto_init_wait(&wait);
+ acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_req_done, &wait);
+
+ ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
+ if (ret < 0) {
+ erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
+ ret, rq->inputsize, rq->pageofs_in, rq->outputsize);
+ ret = -EIO;
+ } else
+ ret = 0;
+
+ sg_free_table(&st_dst);
+failed_dst_alloc:
+ sg_free_table(&st_src);
+failed_src_alloc:
+ acomp_request_free(req);
+ return ret;
+}
+
+int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
+ struct crypto_acomp *erofs_tfm, struct page **pgpl)
+{
+ int i;
+
+ for (i = 0; i < rq->outpages; i++) {
+ struct page *const page = rq->out[i];
+ struct page *victim;
+
+ if (!page) {
+ victim = __erofs_allocpage(pgpl, rq->gfp, true);
+ if (!victim)
+ return -ENOMEM;
+ set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
+ rq->out[i] = victim;
+ }
+ }
+
+ return z_erofs_crypto_decompress_mem(rq, erofs_tfm);
+}
+
+struct crypto_acomp *z_erofs_crypto_get_engine(int type)
+{
+ int i = 0;
+
+ while (z_erofs_crypto[type][i].crypto_name) {
+ if (z_erofs_crypto[type][i].enabled)
+ return z_erofs_crypto[type][i].erofs_tfm;
+ i++;
+ }
+ return NULL;
+}
+
+static int z_erofs_crypto_get_compress_type(const char *name)
+{
+ int i, j;
+
+ for (i = 0; i < Z_EROFS_COMPRESSION_MAX; i++) {
+ j = 0;
+ while (z_erofs_crypto[i][j].crypto_name) {
+ if (!strcmp(name, z_erofs_crypto[i][j].crypto_name))
+ return i;
+
+ j++;
+ }
+ }
+ return -EINVAL;
+}
+
+int z_erofs_crypto_enable_engine(const char *name)
+{
+ int i = 0, type;
+
+ type = z_erofs_crypto_get_compress_type(name);
+ if (type < 0)
+ return -EINVAL;
+
+ while (z_erofs_crypto[type][i].crypto_name) {
+ if (!strcmp(z_erofs_crypto[type][i].crypto_name, name)) {
+ if (z_erofs_crypto[type][i].enabled)
+ break;
+
+ z_erofs_crypto[type][i].erofs_tfm =
+ crypto_alloc_acomp(z_erofs_crypto[type][i].crypto_name, 0, 0);
+ if (IS_ERR(z_erofs_crypto[type][i].erofs_tfm)) {
+ z_erofs_crypto[type][i].erofs_tfm = NULL;
+ return -ENXIO;
+ }
+ z_erofs_crypto[type][i].enabled = true;
+ } else if (z_erofs_crypto[type][i].enabled) {
+ if (z_erofs_crypto[type][i].erofs_tfm)
+ crypto_free_acomp(z_erofs_crypto[type][i].erofs_tfm);
+ z_erofs_crypto[type][i].enabled = false;
+ }
+ i++;
+ }
+ return 0;
+}
+
+void z_erofs_crypto_disable_engine(void)
+{
+ int i = 0, type;
+
+ for (type = 0; type < Z_EROFS_COMPRESSION_MAX; type++) {
+ i = 0;
+ while (z_erofs_crypto[type][i].crypto_name) {
+ if (z_erofs_crypto[type][i].enabled &&
+ z_erofs_crypto[type][i].erofs_tfm) {
+ crypto_free_acomp(z_erofs_crypto[type][i].erofs_tfm);
+ z_erofs_crypto[type][i].erofs_tfm = NULL;
+ z_erofs_crypto[type][i].enabled = false;
+ }
+ i++;
+ }
+ }
+}
+
+int z_erofs_crypto_engine_format(char *buf)
+{
+ int type, i, len = 0;
+
+ for (type = 0; type < Z_EROFS_COMPRESSION_MAX; type++) {
+ i = 0;
+ while (z_erofs_crypto[type][i].crypto_name) {
+ if (z_erofs_crypto[type][i].enabled)
+ len += scnprintf(buf + len, PATH_MAX - len, "%s ",
+ z_erofs_crypto[type][i].crypto_name);
+ i++;
+ }
+ }
+ return len;
+}
+
+struct z_erofs_crypto_engine z_erofs_crypto[Z_EROFS_COMPRESSION_MAX][2] = {
+ [Z_EROFS_COMPRESSION_LZ4] = {
+ (struct z_erofs_crypto_engine) {NULL},
+ (struct z_erofs_crypto_engine) {NULL},
+ },
+ [Z_EROFS_COMPRESSION_LZMA] = {
+ (struct z_erofs_crypto_engine) {NULL},
+ (struct z_erofs_crypto_engine) {NULL},
+ },
+ [Z_EROFS_COMPRESSION_DEFLATE] = {
+ (struct z_erofs_crypto_engine) {
+ .crypto_name = "qat_deflate",
+ .enabled = false,
+ .erofs_tfm = NULL,
+ },
+ (struct z_erofs_crypto_engine) {NULL},
+ },
+ [Z_EROFS_COMPRESSION_ZSTD] = {
+ (struct z_erofs_crypto_engine) {NULL},
+ (struct z_erofs_crypto_engine) {NULL},
+ },
+};
diff --git a/fs/erofs/decompressor_deflate.c b/fs/erofs/decompressor_deflate.c
index c6908a487054..bebbf1eccd3d 100644
--- a/fs/erofs/decompressor_deflate.c
+++ b/fs/erofs/decompressor_deflate.c
@@ -97,7 +97,7 @@ static int z_erofs_load_deflate_config(struct super_block *sb,
return -ENOMEM;
}
-static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
+static int __z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
struct page **pgpl)
{
struct super_block *sb = rq->sb;
@@ -178,6 +178,20 @@ static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
return err;
}
+static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
+ struct page **pgpl)
+{
+#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
+ struct crypto_acomp *erofs_tfm = NULL;
+
+ erofs_tfm = z_erofs_crypto_get_engine(Z_EROFS_COMPRESSION_DEFLATE);
+ if (erofs_tfm && !rq->partial_decoding)
+ return z_erofs_crypto_decompress(rq, erofs_tfm, pgpl);
+ else
+#endif
+ return __z_erofs_deflate_decompress(rq, pgpl);
+}
+
const struct z_erofs_decompressor z_erofs_deflate_decomp = {
.config = z_erofs_load_deflate_config,
.decompress = z_erofs_deflate_decompress,
diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c
index dad4e6c6c155..00123c4d3196 100644
--- a/fs/erofs/sysfs.c
+++ b/fs/erofs/sysfs.c
@@ -7,12 +7,14 @@
#include <linux/kobject.h>
#include "internal.h"
+#include "compress.h"
enum {
attr_feature,
attr_drop_caches,
attr_pointer_ui,
attr_pointer_bool,
+ attr_crypto,
};
enum {
@@ -60,6 +62,9 @@ static struct erofs_attr erofs_attr_##_name = { \
EROFS_ATTR_RW_UI(sync_decompress, erofs_mount_opts);
EROFS_ATTR_FUNC(drop_caches, 0200);
#endif
+#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
+EROFS_ATTR_FUNC(crypto, 0644);
+#endif
static struct attribute *erofs_attrs[] = {
#ifdef CONFIG_EROFS_FS_ZIP
@@ -95,6 +100,9 @@ static struct attribute *erofs_feat_attrs[] = {
ATTR_LIST(fragments),
ATTR_LIST(dedupe),
ATTR_LIST(48bit),
+#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
+ ATTR_LIST(crypto),
+#endif
NULL,
};
ATTRIBUTE_GROUPS(erofs_feat);
@@ -128,6 +136,10 @@ static ssize_t erofs_attr_show(struct kobject *kobj,
if (!ptr)
return 0;
return sysfs_emit(buf, "%d\n", *(bool *)ptr);
+#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
+ case attr_crypto:
+ return z_erofs_crypto_engine_format(buf);
+#endif
}
return 0;
}
@@ -141,6 +153,10 @@ static ssize_t erofs_attr_store(struct kobject *kobj, struct attribute *attr,
unsigned char *ptr = __struct_ptr(sbi, a->struct_type, a->offset);
unsigned long t;
int ret;
+#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
+ char *crypto_name;
+ size_t sz;
+#endif
switch (a->attr_id) {
case attr_pointer_ui:
@@ -181,6 +197,26 @@ static ssize_t erofs_attr_store(struct kobject *kobj, struct attribute *attr,
if (t & 1)
invalidate_mapping_pages(MNGD_MAPPING(sbi), 0, -1);
return len;
+#endif
+#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
+ case attr_crypto:
+ buf = skip_spaces(buf);
+ sz = strlen(buf);
+ crypto_name = kstrdup(buf, GFP_KERNEL);
+ if (!crypto_name)
+ return -ENOMEM;
+
+ /* ignore trailing newline */
+ if (sz > 0 && crypto_name[sz - 1] == '\n')
+ crypto_name[sz - 1] = 0x00;
+
+ if (strlen(crypto_name) > 0) {
+ ret = z_erofs_crypto_enable_engine(crypto_name);
+ if (ret < 0)
+ return ret;
+ } else
+ z_erofs_crypto_disable_engine();
+ return len;
#endif
}
return 0;
--
2.31.1On 2025/5/16 16:26, Bo Liu wrote:
> [外部邮件] 此邮件来源于小米公司外部，请谨慎处理。若对邮件安全性存疑，请将邮件转发给misec@xiaomi.com进行反馈
>
> This patch introdueces the use of the Intel QAT to decompress compressed
> data in the EROFS filesystem, aiming to improve the decompression speed
> of compressed datea.
>
> We created a 285MiB compressed file and then used the following command to
> create EROFS images with different cluster size.
> # mkfs.erofs -zdeflate,level=9 -C16384
>
> fio command was used to test random read and small random read(~5%) and
> sequential read performance.
> # fio -filename=testfile -bs=4k -rw=read -name=job1
> # fio -filename=testfile -bs=4k -rw=randread -name=job1
> # fio -filename=testfile -bs=4k -rw=randread --io_size=14m -name=job1
>
> Here are some performance numbers for reference:
>
> Processors: Intel(R) Xeon(R) 6766E(144 core)
> Memory: 521 GiB
>
> |-----------------------------------------------------------------------------|
> | | Cluster size | sequential read | randread | small randread(5%) |
> |-----------|--------------|-----------------|-----------|--------------------|
> | Intel QAT | 4096 | 538 MiB/s | 112 MiB/s | 20.76 MiB/s |
> | Intel QAT | 16384 | 699 MiB/s | 158 MiB/s | 21.02 MiB/s |
> | Intel QAT | 65536 | 917 MiB/s | 278 MiB/s | 20.90 MiB/s |
> | Intel QAT | 131072 | 1056 MiB/s | 351 MiB/s | 23.36 MiB/s |
> | Intel QAT | 262144 | 1145 MiB/s | 431 MiB/s | 26.66 MiB/s |
> | deflate | 4096 | 499 MiB/s | 108 MiB/s | 21.50 MiB/s |
> | deflate | 16384 | 422 MiB/s | 125 MiB/s | 18.94 MiB/s |
> | deflate | 65536 | 452 MiB/s | 159 MiB/s | 13.02 MiB/s |
> | deflate | 131072 | 452 MiB/s | 177 MiB/s | 11.44 MiB/s |
> | deflate | 262144 | 466 MiB/s | 194 MiB/s | 10.60 MiB/s |
>
> Signed-off-by: Bo Liu <liubo03@inspur.com>
> ---
> v1: https://lore.kernel.org/linux-erofs/20250410042048.3044-1-liubo03@inspur.com/
> v2: https://lore.kernel.org/linux-erofs/20250410042048.3044-1-liubo03@inspur.com/T/#t
>
> Changes since v2:
> - Add a new kernel config option CONFIG_EROFS_FS_ZIP_CRYPTO.
> - Modify the sysfs interface to merge enable and disable into
> a single entry point, and add a read function to view the current state.
> - Optimize the relevant code,use the sg_alloc_table_from_pages_segment interface.
>
> fs/erofs/Kconfig | 14 +++
> fs/erofs/Makefile | 1 +
> fs/erofs/compress.h | 14 +++
> fs/erofs/decompressor_crypto.c | 199 ++++++++++++++++++++++++++++++++
> fs/erofs/decompressor_deflate.c | 16 ++-
> fs/erofs/sysfs.c | 36 ++++++
> 6 files changed, 279 insertions(+), 1 deletion(-)
> create mode 100644 fs/erofs/decompressor_crypto.c
>
> diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig
> index 331e49cd1b8d..db49ae3c1922 100644
> --- a/fs/erofs/Kconfig
> +++ b/fs/erofs/Kconfig
> @@ -144,6 +144,20 @@ config EROFS_FS_ZIP_ZSTD
>
> If unsure, say N.
>
> +config EROFS_FS_ZIP_CRYPTO
> + bool "EROFS hardware decompression support (crypto interface)"
> + depends on EROFS_FS_ZIP
> + help
> + Saying Y here includes support for reading EROFS file systems
> + containing crypto compressed data. It gives better decompression
> + speed than the software-implemented compression, and it costs
> + lower CPU overhead.
> +
> + Crypto support is an experimental feature for now and so most
> + file systems will be readable without selecting this option.
> +
> + If unsure, say N.
> +
> config EROFS_FS_ONDEMAND
> bool "EROFS fscache-based on-demand read support (deprecated)"
> depends on EROFS_FS
> diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile
> index 4331d53c7109..247b263eb667 100644
> --- a/fs/erofs/Makefile
> +++ b/fs/erofs/Makefile
> @@ -7,5 +7,6 @@ erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o zutil.o
> erofs-$(CONFIG_EROFS_FS_ZIP_LZMA) += decompressor_lzma.o
> erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o
> erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o
> +erofs-$(CONFIG_EROFS_FS_ZIP_CRYPTO) += decompressor_crypto.o
> erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o
> erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o
> diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h
> index 2704d7a592a5..356ae2473bd5 100644
> --- a/fs/erofs/compress.h
> +++ b/fs/erofs/compress.h
> @@ -70,10 +70,24 @@ struct z_erofs_stream_dctx {
> bool bounced; /* is the bounce buffer used now? */
> };
>
> +struct z_erofs_crypto_engine {
> + char *crypto_name;
> + bool enabled;
> + struct crypto_acomp *erofs_tfm;
> +};
> +
> +extern struct z_erofs_crypto_engine z_erofs_crypto[Z_EROFS_COMPRESSION_MAX][2];
> +
> int z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, void **dst,
> void **src, struct page **pgpl);
> int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,
> unsigned int padbufsize);
> int __init z_erofs_init_decompressor(void);
> void z_erofs_exit_decompressor(void);
> +int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
> + struct crypto_acomp *erofs_tfm, struct page **pgpl);
> +struct crypto_acomp *z_erofs_crypto_get_engine(int type);
> +int z_erofs_crypto_enable_engine(const char *name);
> +void z_erofs_crypto_disable_engine(void);
> +int z_erofs_crypto_engine_format(char *buf);
> #endif
> diff --git a/fs/erofs/decompressor_crypto.c b/fs/erofs/decompressor_crypto.c
> new file mode 100644
> index 000000000000..55f6a9e6dcf6
> --- /dev/null
> +++ b/fs/erofs/decompressor_crypto.c
> @@ -0,0 +1,199 @@
> +// SPDX-License-Identifier: GPL-2.0-or-later
> +#include <linux/scatterlist.h>
> +#include <crypto/acompress.h>
> +
> +#include "compress.h"
> +
> +static int z_erofs_crypto_decompress_mem(struct z_erofs_decompress_req *rq,
> + struct crypto_acomp *erofs_tfm)
> +{
> + struct sg_table st_src, st_dst;
> + struct acomp_req *req;
> + struct crypto_wait wait;
> + u8 *headpage;
> + int ret;
> +
> + headpage = kmap_local_page(*rq->in);
> + ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
> + min_t(unsigned int, rq->inputsize,
> + rq->sb->s_blocksize - rq->pageofs_in));
> + kunmap_local(headpage);
> + if (ret)
> + return ret;
> +
> + req = acomp_request_alloc(erofs_tfm);
> + if (!req) {
> + erofs_err(rq->sb, "failed to alloc decompress request");
> + return -ENOMEM;
> + }
> +
> + ret = sg_alloc_table_from_pages_segment(&st_src,
> + rq->in, rq->inpages, rq->pageofs_in,
> + rq->inputsize, UINT_MAX, GFP_KERNEL);
> + if (ret < 0)
> + goto failed_src_alloc;
> +
> + ret = sg_alloc_table_from_pages_segment(&st_dst,
> + rq->out, rq->outpages, rq->pageofs_out,
> + rq->outputsize, UINT_MAX, GFP_KERNEL);
> + if (ret < 0)
> + goto failed_dst_alloc;
> +
> + acomp_request_set_params(req, st_src.sgl,
> + st_dst.sgl, rq->inputsize, rq->outputsize);
> +
> + crypto_init_wait(&wait);
> + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
> + crypto_req_done, &wait);
> +
> + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
> + if (ret < 0) {
> + erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
> + ret, rq->inputsize, rq->pageofs_in, rq->outputsize);
> + ret = -EIO;
> + } else
> + ret = 0;
> +
> + sg_free_table(&st_dst);
> +failed_dst_alloc:
> + sg_free_table(&st_src);
> +failed_src_alloc:
> + acomp_request_free(req);
> + return ret;
> +}
> +
> +int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
> + struct crypto_acomp *erofs_tfm, struct page **pgpl)
> +{
> + int i;
> +
> + for (i = 0; i < rq->outpages; i++) {
> + struct page *const page = rq->out[i];
> + struct page *victim;
> +
> + if (!page) {
> + victim = __erofs_allocpage(pgpl, rq->gfp, true);
> + if (!victim)
> + return -ENOMEM;
> + set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
> + rq->out[i] = victim;
> + }
> + }
> +
> + return z_erofs_crypto_decompress_mem(rq, erofs_tfm);
> +}
> +
> +struct crypto_acomp *z_erofs_crypto_get_engine(int type)
> +{
> + int i = 0;
> +
> + while (z_erofs_crypto[type][i].crypto_name) {
> + if (z_erofs_crypto[type][i].enabled)
> + return z_erofs_crypto[type][i].erofs_tfm;
> + i++;
> + }
> + return NULL;
> +}
> +
> +static int z_erofs_crypto_get_compress_type(const char *name)
> +{
> + int i, j;
> +
> + for (i = 0; i < Z_EROFS_COMPRESSION_MAX; i++) {
> + j = 0;
> + while (z_erofs_crypto[i][j].crypto_name) {
Is it possible to use z_erofs_crypto_engine itself instead of crypto_name to
check if there is a supported compression engine, like:
#define Z_EROFS_CRYPTO_ENGINE_MAX 1
struct z_erofs_crypto_engine *z_erofs_crypto[Z_EROFS_COMPRESSION_MAX][Z_EROFS_CRYPTO_ENGINE_MAX] = {
[Z_EROFS_COMPRESSION_LZ4] = { },
[Z_EROFS_COMPRESSION_LZMA] = { },
[Z_EROFS_COMPRESSION_DEFLATE] = {
&(struct z_erofs_crypto_engine) {
.crypto_name = "qat_deflate",
.enabled = false,
.erofs_tfm = NULL,
}
},
[Z_EROFS_COMPRESSION_ZSTD] = {
},
};
for (i = 0; z_erofs_crypto[type][i]; ++i) {
...
}
> + if (!strcmp(name, z_erofs_crypto[i][j].crypto_name))
> + return i;
> +
> + j++;
> + }
> + }
> + return -EINVAL;
> +}
> +
> +int z_erofs_crypto_enable_engine(const char *name)
> +{
> + int i = 0, type;
> +
> + type = z_erofs_crypto_get_compress_type(name);
> + if (type < 0)
> + return -EINVAL;
> +
> + while (z_erofs_crypto[type][i].crypto_name) {
> + if (!strcmp(z_erofs_crypto[type][i].crypto_name, name)) {
> + if (z_erofs_crypto[type][i].enabled)
> + break;
Maybe an error is needed to inform duplicate enable.
> +
> + z_erofs_crypto[type][i].erofs_tfm =
> + crypto_alloc_acomp(z_erofs_crypto[type][i].crypto_name, 0, 0);
> + if (IS_ERR(z_erofs_crypto[type][i].erofs_tfm)) {
> + z_erofs_crypto[type][i].erofs_tfm = NULL;
> + return -ENXIO;
> + }
> + z_erofs_crypto[type][i].enabled = true;
> + } else if (z_erofs_crypto[type][i].enabled) {
> + if (z_erofs_crypto[type][i].erofs_tfm)
> + crypto_free_acomp(z_erofs_crypto[type][i].erofs_tfm);
> + z_erofs_crypto[type][i].enabled = false;
This may be concurrent with z_erofs_deflate_decompress, so it should be
disabled first and may require a memory barrier.
> + }
> + i++;
> + }
> + return 0;
> +}
> +
> +void z_erofs_crypto_disable_engine(void)
> +{
> + int i = 0, type;
> +
> + for (type = 0; type < Z_EROFS_COMPRESSION_MAX; type++) {
> + i = 0;
> + while (z_erofs_crypto[type][i].crypto_name) {
> + if (z_erofs_crypto[type][i].enabled &&
> + z_erofs_crypto[type][i].erofs_tfm) {
> + crypto_free_acomp(z_erofs_crypto[type][i].erofs_tfm);
> + z_erofs_crypto[type][i].erofs_tfm = NULL;
> + z_erofs_crypto[type][i].enabled = false;
> + }
> + i++;
> + }
> + }
> +}
> +
> +int z_erofs_crypto_engine_format(char *buf)
> +{
> + int type, i, len = 0;
> +
> + for (type = 0; type < Z_EROFS_COMPRESSION_MAX; type++) {
> + i = 0;
> + while (z_erofs_crypto[type][i].crypto_name) {
> + if (z_erofs_crypto[type][i].enabled)
> + len += scnprintf(buf + len, PATH_MAX - len, "%s ",
> + z_erofs_crypto[type][i].crypto_name);
> + i++;
> + }
> + }
> + return len;
> +}
> +
> +struct z_erofs_crypto_engine z_erofs_crypto[Z_EROFS_COMPRESSION_MAX][2] = {
> + [Z_EROFS_COMPRESSION_LZ4] = {
> + (struct z_erofs_crypto_engine) {NULL},
> + (struct z_erofs_crypto_engine) {NULL},
> + },
> + [Z_EROFS_COMPRESSION_LZMA] = {
> + (struct z_erofs_crypto_engine) {NULL},
> + (struct z_erofs_crypto_engine) {NULL},
> + },
> + [Z_EROFS_COMPRESSION_DEFLATE] = {
> + (struct z_erofs_crypto_engine) {
> + .crypto_name = "qat_deflate",
> + .enabled = false,
> + .erofs_tfm = NULL,
> + },
> + (struct z_erofs_crypto_engine) {NULL},
> + },
> + [Z_EROFS_COMPRESSION_ZSTD] = {
> + (struct z_erofs_crypto_engine) {NULL},
> + (struct z_erofs_crypto_engine) {NULL},
> + },
> +};
> diff --git a/fs/erofs/decompressor_deflate.c b/fs/erofs/decompressor_deflate.c
> index c6908a487054..bebbf1eccd3d 100644
> --- a/fs/erofs/decompressor_deflate.c
> +++ b/fs/erofs/decompressor_deflate.c
> @@ -97,7 +97,7 @@ static int z_erofs_load_deflate_config(struct super_block *sb,
> return -ENOMEM;
> }
>
> -static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
> +static int __z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
> struct page **pgpl)
> {
> struct super_block *sb = rq->sb;
> @@ -178,6 +178,20 @@ static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
> return err;
> }
>
> +static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
> + struct page **pgpl)
> +{
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + struct crypto_acomp *erofs_tfm = NULL;
> +
> + erofs_tfm = z_erofs_crypto_get_engine(Z_EROFS_COMPRESSION_DEFLATE);
> + if (erofs_tfm && !rq->partial_decoding)
> + return z_erofs_crypto_decompress(rq, erofs_tfm, pgpl);
> + else
Unnecessary else.
> +#endif
> + return __z_erofs_deflate_decompress(rq, pgpl);
> +}
> +
> const struct z_erofs_decompressor z_erofs_deflate_decomp = {
> .config = z_erofs_load_deflate_config,
> .decompress = z_erofs_deflate_decompress,
> diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c
> index dad4e6c6c155..00123c4d3196 100644
> --- a/fs/erofs/sysfs.c
> +++ b/fs/erofs/sysfs.c
> @@ -7,12 +7,14 @@
> #include <linux/kobject.h>
>
> #include "internal.h"
> +#include "compress.h"
>
> enum {
> attr_feature,
> attr_drop_caches,
> attr_pointer_ui,
> attr_pointer_bool,
> + attr_crypto,
> };
>
> enum {
> @@ -60,6 +62,9 @@ static struct erofs_attr erofs_attr_##_name = { \
> EROFS_ATTR_RW_UI(sync_decompress, erofs_mount_opts);
> EROFS_ATTR_FUNC(drop_caches, 0200);
> #endif
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> +EROFS_ATTR_FUNC(crypto, 0644);
> +#endif
>
> static struct attribute *erofs_attrs[] = {
> #ifdef CONFIG_EROFS_FS_ZIP
> @@ -95,6 +100,9 @@ static struct attribute *erofs_feat_attrs[] = {
> ATTR_LIST(fragments),
> ATTR_LIST(dedupe),
> ATTR_LIST(48bit),
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + ATTR_LIST(crypto),
> +#endif
> NULL,
> };
> ATTRIBUTE_GROUPS(erofs_feat);
> @@ -128,6 +136,10 @@ static ssize_t erofs_attr_show(struct kobject *kobj,
> if (!ptr)
> return 0;
> return sysfs_emit(buf, "%d\n", *(bool *)ptr);
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + case attr_crypto:
> + return z_erofs_crypto_engine_format(buf);
> +#endif
> }
> return 0;
> }
> @@ -141,6 +153,10 @@ static ssize_t erofs_attr_store(struct kobject *kobj, struct attribute *attr,
> unsigned char *ptr = __struct_ptr(sbi, a->struct_type, a->offset);
> unsigned long t;
> int ret;
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + char *crypto_name;
> + size_t sz;
> +#endif
>
> switch (a->attr_id) {
> case attr_pointer_ui:
> @@ -181,6 +197,26 @@ static ssize_t erofs_attr_store(struct kobject *kobj, struct attribute *attr,
> if (t & 1)
> invalidate_mapping_pages(MNGD_MAPPING(sbi), 0, -1);
> return len;
> +#endif
> +#ifdef CONFIG_EROFS_FS_ZIP_CRYPTO
> + case attr_crypto:
> + buf = skip_spaces(buf);
> + sz = strlen(buf);
> + crypto_name = kstrdup(buf, GFP_KERNEL);
> + if (!crypto_name)
> + return -ENOMEM;
> +
> + /* ignore trailing newline */
> + if (sz > 0 && crypto_name[sz - 1] == '\n')
> + crypto_name[sz - 1] = 0x00;
> +
> + if (strlen(crypto_name) > 0) {
> + ret = z_erofs_crypto_enable_engine(crypto_name);
> + if (ret < 0)
> + return ret;
> + } else
> + z_erofs_crypto_disable_engine();
Is it possible to support disable a single engine?
Thanks,
Jianan
> + return len;
> #endif
> }
> return 0;
> --
> 2.31.1
>
>
On Fri, May 16, 2025 at 04:26:34AM -0400, Bo Liu wrote: > +config EROFS_FS_ZIP_CRYPTO > + bool "EROFS hardware decompression support (crypto interface)" > + depends on EROFS_FS_ZIP > + help > + Saying Y here includes support for reading EROFS file systems > + containing crypto compressed data. It gives better decompression > + speed than the software-implemented compression, and it costs > + lower CPU overhead. > + > + Crypto support is an experimental feature for now and so most > + file systems will be readable without selecting this option. > + > + If unsure, say N. I recommend not including the word "crypto" in any user facing part of this. Compression algorithms are not cryptographic algorithms. The fact that the interface to access hardware compression accelerators is currently the "Crypto API" is an implementation quirk. It could be a different interface in the future. Call it something clear like "hardware decompression". - Eric
Hi Eric, On 2025/5/17 00:38, Eric Biggers wrote: > On Fri, May 16, 2025 at 04:26:34AM -0400, Bo Liu wrote: >> +config EROFS_FS_ZIP_CRYPTO >> + bool "EROFS hardware decompression support (crypto interface)" >> + depends on EROFS_FS_ZIP >> + help >> + Saying Y here includes support for reading EROFS file systems >> + containing crypto compressed data. It gives better decompression >> + speed than the software-implemented compression, and it costs >> + lower CPU overhead. >> + >> + Crypto support is an experimental feature for now and so most >> + file systems will be readable without selecting this option. >> + >> + If unsure, say N. > > I recommend not including the word "crypto" in any user facing part of this. > > Compression algorithms are not cryptographic algorithms. The fact that the > interface to access hardware compression accelerators is currently the "Crypto > API" is an implementation quirk. It could be a different interface in the > future. > > Call it something clear like "hardware decompression". Totally agreed on this since it's just an implementation detail, and I will try to make some extra time to polish this commit myself to fix some minor issues later. Thanks, Gao Xiang > > - Eric
From: Bo Liu <liubo03@inspur.com>
This patch introduces the use of the Intel QAT to offload data compression
in the EROFS filesystem, aiming to improve the decompression performance.
A 285MiB dataset is used with the following command to create EROFS images
with different cluster sizes:
$ mkfs.erofs -zdeflate,level=9 -C16384
Fio is used to test the following read patterns:
# fio -filename=testfile -bs=4k -rw=read -name=job1
# fio -filename=testfile -bs=4k -rw=randread -name=job1
# fio -filename=testfile -bs=4k -rw=randread --io_size=14m -name=job1
Here are some performance numbers for reference:
Processors: Intel(R) Xeon(R) 6766E(144 core)
Memory: 521 GiB
|-----------------------------------------------------------------------------|
| | Cluster size | sequential read | randread | small randread(5%) |
|-----------|--------------|-----------------|-----------|--------------------|
| Intel QAT | 4096 | 538 MiB/s | 112 MiB/s | 20.76 MiB/s |
| Intel QAT | 16384 | 699 MiB/s | 158 MiB/s | 21.02 MiB/s |
| Intel QAT | 65536 | 917 MiB/s | 278 MiB/s | 20.90 MiB/s |
| Intel QAT | 131072 | 1056 MiB/s | 351 MiB/s | 23.36 MiB/s |
| Intel QAT | 262144 | 1145 MiB/s | 431 MiB/s | 26.66 MiB/s |
| deflate | 4096 | 499 MiB/s | 108 MiB/s | 21.50 MiB/s |
| deflate | 16384 | 422 MiB/s | 125 MiB/s | 18.94 MiB/s |
| deflate | 65536 | 452 MiB/s | 159 MiB/s | 13.02 MiB/s |
| deflate | 131072 | 452 MiB/s | 177 MiB/s | 11.44 MiB/s |
| deflate | 262144 | 466 MiB/s | 194 MiB/s | 10.60 MiB/s |
Signed-off-by: Bo Liu <liubo03@inspur.com>
Signed-off-by: Gao Xiang <xiang@kernel.org>
---
Hi Bo,
Please test/refine this version and complete sysfs documentation.
Thanks,
Gao Xiang
fs/erofs/Kconfig | 14 +++
fs/erofs/Makefile | 1 +
fs/erofs/compress.h | 10 ++
fs/erofs/decompressor_crypto.c | 184 ++++++++++++++++++++++++++++++++
fs/erofs/decompressor_deflate.c | 20 +++-
fs/erofs/sysfs.c | 35 +++++-
fs/erofs/zdata.c | 1 +
7 files changed, 260 insertions(+), 5 deletions(-)
create mode 100644 fs/erofs/decompressor_crypto.c
diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig
index 8f68ec49ad89..6beeb7063871 100644
--- a/fs/erofs/Kconfig
+++ b/fs/erofs/Kconfig
@@ -144,6 +144,20 @@ config EROFS_FS_ZIP_ZSTD
If unsure, say N.
+config EROFS_FS_ZIP_ACCEL
+ bool "EROFS hardware decompression support"
+ depends on EROFS_FS_ZIP
+ help
+ Saying Y here includes hardware accelerator support for reading
+ EROFS file systems containing compressed data. It gives better
+ decompression speed than the software-implemented decompression, and
+ it costs lower CPU overhead.
+
+ Hardware accelerator support is an experimental feature for now and
+ file systems are still readable without selecting this option.
+
+ If unsure, say N.
+
config EROFS_FS_ONDEMAND
bool "EROFS fscache-based on-demand read support (deprecated)"
depends on EROFS_FS
diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile
index 4331d53c7109..549abc424763 100644
--- a/fs/erofs/Makefile
+++ b/fs/erofs/Makefile
@@ -7,5 +7,6 @@ erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o zutil.o
erofs-$(CONFIG_EROFS_FS_ZIP_LZMA) += decompressor_lzma.o
erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o
erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o
+erofs-$(CONFIG_EROFS_FS_ZIP_ACCEL) += decompressor_crypto.o
erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o
erofs-$(CONFIG_EROFS_FS_ONDEMAND) += fscache.o
diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h
index 2704d7a592a5..2bea4097e0b4 100644
--- a/fs/erofs/compress.h
+++ b/fs/erofs/compress.h
@@ -76,4 +76,14 @@ int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,
unsigned int padbufsize);
int __init z_erofs_init_decompressor(void);
void z_erofs_exit_decompressor(void);
+int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
+ struct page **pgpl);
+int z_erofs_crypto_enable_engine(const char *name, int len);
+#ifdef CONFIG_EROFS_FS_ZIP_ACCEL
+void z_erofs_crypto_disable_all_engines(void);
+int z_erofs_crypto_show_engines(char *buf, int size, char sep);
+#else
+static inline void z_erofs_crypto_disable_all_engines(void) {}
+static inline int z_erofs_crypto_show_engines(char *buf, int size, char sep) {}
+#endif
#endif
diff --git a/fs/erofs/decompressor_crypto.c b/fs/erofs/decompressor_crypto.c
new file mode 100644
index 000000000000..ba0f46f6ef12
--- /dev/null
+++ b/fs/erofs/decompressor_crypto.c
@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/scatterlist.h>
+#include <crypto/acompress.h>
+#include "compress.h"
+
+static int __z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
+ struct crypto_acomp *tfm)
+{
+ struct sg_table st_src, st_dst;
+ struct acomp_req *req;
+ struct crypto_wait wait;
+ u8 *headpage;
+ int ret;
+
+ headpage = kmap_local_page(*rq->in);
+ ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
+ min_t(unsigned int, rq->inputsize,
+ rq->sb->s_blocksize - rq->pageofs_in));
+ kunmap_local(headpage);
+ if (ret)
+ return ret;
+
+ req = acomp_request_alloc(tfm);
+ if (!req) {
+ erofs_err(rq->sb, "failed to alloc decompress request");
+ return -ENOMEM;
+ }
+
+ ret = sg_alloc_table_from_pages_segment(&st_src, rq->in, rq->inpages,
+ rq->pageofs_in, rq->inputsize, UINT_MAX, GFP_KERNEL);
+ if (ret < 0)
+ goto failed_src_alloc;
+
+ ret = sg_alloc_table_from_pages_segment(&st_dst, rq->out, rq->outpages,
+ rq->pageofs_out, rq->outputsize, UINT_MAX, GFP_KERNEL);
+ if (ret < 0)
+ goto failed_dst_alloc;
+
+ acomp_request_set_params(req, st_src.sgl,
+ st_dst.sgl, rq->inputsize, rq->outputsize);
+
+ crypto_init_wait(&wait);
+ acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_req_done, &wait);
+
+ ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
+ if (ret) {
+ erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
+ ret, rq->inputsize, rq->pageofs_in, rq->outputsize);
+ ret = -EIO;
+ }
+
+ sg_free_table(&st_dst);
+failed_dst_alloc:
+ sg_free_table(&st_src);
+failed_src_alloc:
+ acomp_request_free(req);
+ return ret;
+}
+
+struct z_erofs_crypto_engine {
+ char *crypto_name;
+ struct crypto_acomp *tfm;
+};
+
+struct z_erofs_crypto_engine *z_erofs_crypto[Z_EROFS_COMPRESSION_MAX] = {
+ [Z_EROFS_COMPRESSION_LZ4] = (struct z_erofs_crypto_engine[]) {
+ {},
+ },
+ [Z_EROFS_COMPRESSION_LZMA] = (struct z_erofs_crypto_engine[]) {
+ {},
+ },
+ [Z_EROFS_COMPRESSION_DEFLATE] = (struct z_erofs_crypto_engine[]) {
+ { .crypto_name = "qat_deflate", },
+ {},
+ },
+ [Z_EROFS_COMPRESSION_ZSTD] = (struct z_erofs_crypto_engine[]) {
+ {},
+ },
+};
+
+static DECLARE_RWSEM(z_erofs_crypto_rwsem);
+
+static struct crypto_acomp *z_erofs_crypto_get_engine(int alg)
+{
+ struct z_erofs_crypto_engine *e;
+
+ for (e = z_erofs_crypto[alg]; e->crypto_name; ++e)
+ if (e->tfm)
+ return e->tfm;
+ return NULL;
+}
+
+int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq,
+ struct page **pgpl)
+{
+ struct crypto_acomp *tfm;
+ int i, err;
+
+ down_read(&z_erofs_crypto_rwsem);
+ tfm = z_erofs_crypto_get_engine(rq->alg);
+ if (!tfm) {
+ err = -EOPNOTSUPP;
+ goto out;
+ }
+
+ for (i = 0; i < rq->outpages; i++) {
+ struct page *const page = rq->out[i];
+ struct page *victim;
+
+ if (!page) {
+ victim = __erofs_allocpage(pgpl, rq->gfp, true);
+ if (!victim) {
+ err = -ENOMEM;
+ goto out;
+ }
+ set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
+ rq->out[i] = victim;
+ }
+ }
+ err = __z_erofs_crypto_decompress(rq, tfm);
+out:
+ up_read(&z_erofs_crypto_rwsem);
+ return err;
+}
+
+int z_erofs_crypto_enable_engine(const char *name, int len)
+{
+ struct z_erofs_crypto_engine *e;
+ struct crypto_acomp *tfm;
+ int alg;
+
+ down_write(&z_erofs_crypto_rwsem);
+ for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) {
+ for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) {
+ if (!strncmp(name, e->crypto_name, len)) {
+ if (e->tfm)
+ break;
+ tfm = crypto_alloc_acomp(e->crypto_name, 0, 0);
+ if (IS_ERR(tfm)) {
+ up_write(&z_erofs_crypto_rwsem);
+ return -EOPNOTSUPP;
+ }
+ e->tfm = tfm;
+ break;
+ }
+ }
+ }
+ up_write(&z_erofs_crypto_rwsem);
+ return 0;
+}
+
+void z_erofs_crypto_disable_all_engines(void)
+{
+ struct z_erofs_crypto_engine *e;
+ int alg;
+
+ down_write(&z_erofs_crypto_rwsem);
+ for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) {
+ for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) {
+ if (!e->tfm)
+ continue;
+ crypto_free_acomp(e->tfm);
+ e->tfm = NULL;
+ }
+ }
+ up_write(&z_erofs_crypto_rwsem);
+}
+
+int z_erofs_crypto_show_engines(char *buf, int size, char sep)
+{
+ struct z_erofs_crypto_engine *e;
+ int alg, len = 0;
+
+ for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) {
+ for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) {
+ if (!e->tfm)
+ continue;
+ len += scnprintf(buf + len, size - len, "%s%c",
+ e->crypto_name, sep);
+ }
+ }
+ return len;
+}
diff --git a/fs/erofs/decompressor_deflate.c b/fs/erofs/decompressor_deflate.c
index c6908a487054..6909b2d529c7 100644
--- a/fs/erofs/decompressor_deflate.c
+++ b/fs/erofs/decompressor_deflate.c
@@ -97,8 +97,8 @@ static int z_erofs_load_deflate_config(struct super_block *sb,
return -ENOMEM;
}
-static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
- struct page **pgpl)
+static int __z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
+ struct page **pgpl)
{
struct super_block *sb = rq->sb;
struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 };
@@ -178,6 +178,22 @@ static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
return err;
}
+static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
+ struct page **pgpl)
+{
+#ifdef CONFIG_EROFS_FS_ZIP_ACCEL
+ int err;
+
+ if (!rq->partial_decoding) {
+ err = z_erofs_crypto_decompress(rq, pgpl);
+ if (err != -EOPNOTSUPP)
+ return err;
+
+ }
+#endif
+ return __z_erofs_deflate_decompress(rq, pgpl);
+}
+
const struct z_erofs_decompressor z_erofs_deflate_decomp = {
.config = z_erofs_load_deflate_config,
.decompress = z_erofs_deflate_decompress,
diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c
index dad4e6c6c155..4c0ad4b93161 100644
--- a/fs/erofs/sysfs.c
+++ b/fs/erofs/sysfs.c
@@ -7,12 +7,14 @@
#include <linux/kobject.h>
#include "internal.h"
+#include "compress.h"
enum {
attr_feature,
attr_drop_caches,
attr_pointer_ui,
attr_pointer_bool,
+ attr_accel,
};
enum {
@@ -60,14 +62,25 @@ static struct erofs_attr erofs_attr_##_name = { \
EROFS_ATTR_RW_UI(sync_decompress, erofs_mount_opts);
EROFS_ATTR_FUNC(drop_caches, 0200);
#endif
+#ifdef CONFIG_EROFS_FS_ZIP_ACCEL
+EROFS_ATTR_FUNC(accel, 0644);
+#endif
-static struct attribute *erofs_attrs[] = {
+static struct attribute *erofs_sb_attrs[] = {
#ifdef CONFIG_EROFS_FS_ZIP
ATTR_LIST(sync_decompress),
ATTR_LIST(drop_caches),
#endif
NULL,
};
+ATTRIBUTE_GROUPS(erofs_sb);
+
+static struct attribute *erofs_attrs[] = {
+#ifdef CONFIG_EROFS_FS_ZIP_ACCEL
+ ATTR_LIST(accel),
+#endif
+ NULL,
+};
ATTRIBUTE_GROUPS(erofs);
/* Features this copy of erofs supports */
@@ -128,12 +141,14 @@ static ssize_t erofs_attr_show(struct kobject *kobj,
if (!ptr)
return 0;
return sysfs_emit(buf, "%d\n", *(bool *)ptr);
+ case attr_accel:
+ return z_erofs_crypto_show_engines(buf, PAGE_SIZE, '\n');
}
return 0;
}
static ssize_t erofs_attr_store(struct kobject *kobj, struct attribute *attr,
- const char *buf, size_t len)
+ const char *buf, size_t len)
{
struct erofs_sb_info *sbi = container_of(kobj, struct erofs_sb_info,
s_kobj);
@@ -181,6 +196,19 @@ static ssize_t erofs_attr_store(struct kobject *kobj, struct attribute *attr,
if (t & 1)
invalidate_mapping_pages(MNGD_MAPPING(sbi), 0, -1);
return len;
+#endif
+#ifdef CONFIG_EROFS_FS_ZIP_ACCEL
+ case attr_accel:
+ buf = skip_spaces(buf);
+ z_erofs_crypto_disable_all_engines();
+ while (*buf) {
+ t = strcspn(buf, "\n");
+ ret = z_erofs_crypto_enable_engine(buf, t);
+ if (ret < 0)
+ return ret;
+ buf += buf[t] == '\n' ? t + 1 : t;
+ }
+ return len;
#endif
}
return 0;
@@ -199,12 +227,13 @@ static const struct sysfs_ops erofs_attr_ops = {
};
static const struct kobj_type erofs_sb_ktype = {
- .default_groups = erofs_groups,
+ .default_groups = erofs_sb_groups,
.sysfs_ops = &erofs_attr_ops,
.release = erofs_sb_release,
};
static const struct kobj_type erofs_ktype = {
+ .default_groups = erofs_groups,
.sysfs_ops = &erofs_attr_ops,
};
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index ab61c84d47cd..fe8071844724 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -441,6 +441,7 @@ void z_erofs_exit_subsystem(void)
z_erofs_destroy_pcpu_workers();
destroy_workqueue(z_erofs_workqueue);
z_erofs_destroy_pcluster_pool();
+ z_erofs_crypto_disable_all_engines();
z_erofs_exit_decompressor();
}
--
2.43.5© 2016 - 2025 Red Hat, Inc.