When utilized it dodges strlen() in vfs_readlink(), giving about 1.5%
speed up when issuing readlink on /initrd.img on ext4.
Filesystems opt in by calling inode_set_cached_link() when creating an
inode.
The size is stored in what used to be a 4-byte hole. If necessary the
field can be made smaller and converted into a union with something not
used with symlinks.
Churn-wise the current readlink_copy() helper is patched to accept the
size instead of calculating it.
Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
---
fs/namei.c | 34 +++++++++++++++++++---------------
fs/proc/namespaces.c | 2 +-
include/linux/fs.h | 12 ++++++++++--
security/apparmor/apparmorfs.c | 2 +-
4 files changed, 31 insertions(+), 19 deletions(-)
diff --git a/fs/namei.c b/fs/namei.c
index 9d30c7aa9aa6..e56c29a22d26 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -5272,19 +5272,16 @@ SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newna
getname(newname), 0);
}
-int readlink_copy(char __user *buffer, int buflen, const char *link)
+int readlink_copy(char __user *buffer, int buflen, const char *link, int linklen)
{
- int len = PTR_ERR(link);
- if (IS_ERR(link))
- goto out;
+ int copylen;
- len = strlen(link);
- if (len > (unsigned) buflen)
- len = buflen;
- if (copy_to_user(buffer, link, len))
- len = -EFAULT;
-out:
- return len;
+ copylen = linklen;
+ if (unlikely(copylen > (unsigned) buflen))
+ copylen = buflen;
+ if (copy_to_user(buffer, link, copylen))
+ copylen = -EFAULT;
+ return copylen;
}
/**
@@ -5304,6 +5301,9 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
const char *link;
int res;
+ if (inode->i_opflags & IOP_CACHED_LINK)
+ return readlink_copy(buffer, buflen, inode->i_link, inode->i_linklen);
+
if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
if (unlikely(inode->i_op->readlink))
return inode->i_op->readlink(dentry, buffer, buflen);
@@ -5322,7 +5322,7 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
if (IS_ERR(link))
return PTR_ERR(link);
}
- res = readlink_copy(buffer, buflen, link);
+ res = readlink_copy(buffer, buflen, link, strlen(link));
do_delayed_call(&done);
return res;
}
@@ -5391,10 +5391,14 @@ EXPORT_SYMBOL(page_put_link);
int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
{
+ const char *link;
+ int res;
+
DEFINE_DELAYED_CALL(done);
- int res = readlink_copy(buffer, buflen,
- page_get_link(dentry, d_inode(dentry),
- &done));
+ link = page_get_link(dentry, d_inode(dentry), &done);
+ res = PTR_ERR(link);
+ if (!IS_ERR(link))
+ res = readlink_copy(buffer, buflen, link, strlen(link));
do_delayed_call(&done);
return res;
}
diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c
index 8e159fc78c0a..c610224faf10 100644
--- a/fs/proc/namespaces.c
+++ b/fs/proc/namespaces.c
@@ -83,7 +83,7 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl
if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
res = ns_get_name(name, sizeof(name), task, ns_ops);
if (res >= 0)
- res = readlink_copy(buffer, buflen, name);
+ res = readlink_copy(buffer, buflen, name, strlen(name));
}
put_task_struct(task);
return res;
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 972147da71f9..30e332fb399d 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -626,6 +626,7 @@ is_uncached_acl(struct posix_acl *acl)
#define IOP_XATTR 0x0008
#define IOP_DEFAULT_READLINK 0x0010
#define IOP_MGTIME 0x0020
+#define IOP_CACHED_LINK 0x0040
/*
* Keep mostly read-only and often accessed (especially for
@@ -686,7 +687,7 @@ struct inode {
/* Misc */
u32 i_state;
- /* 32-bit hole */
+ int i_linklen; /* for symlinks */
struct rw_semaphore i_rwsem;
unsigned long dirtied_when; /* jiffies of first dirtying */
@@ -749,6 +750,13 @@ struct inode {
void *i_private; /* fs or device private pointer */
} __randomize_layout;
+static inline void inode_set_cached_link(struct inode *inode, char *link, int linklen)
+{
+ inode->i_link = link;
+ inode->i_linklen = linklen;
+ inode->i_opflags |= IOP_CACHED_LINK;
+}
+
/*
* Get bit address from inode->i_state to use with wait_var_event()
* infrastructre.
@@ -3351,7 +3359,7 @@ extern const struct file_operations generic_ro_fops;
#define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))
-extern int readlink_copy(char __user *, int, const char *);
+extern int readlink_copy(char __user *, int, const char *, int);
extern int page_readlink(struct dentry *, char __user *, int);
extern const char *page_get_link(struct dentry *, struct inode *,
struct delayed_call *);
diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c
index 01b923d97a44..60959cfba672 100644
--- a/security/apparmor/apparmorfs.c
+++ b/security/apparmor/apparmorfs.c
@@ -2611,7 +2611,7 @@ static int policy_readlink(struct dentry *dentry, char __user *buffer,
res = snprintf(name, sizeof(name), "%s:[%lu]", AAFS_NAME,
d_inode(dentry)->i_ino);
if (res > 0 && res < sizeof(name))
- res = readlink_copy(buffer, buflen, name);
+ res = readlink_copy(buffer, buflen, name, strlen(name));
else
res = -ENOENT;
--
2.43.0On 2024/11/19 17:45, Mateusz Guzik wrote:
> When utilized it dodges strlen() in vfs_readlink(), giving about 1.5%
> speed up when issuing readlink on /initrd.img on ext4.
>
> Filesystems opt in by calling inode_set_cached_link() when creating an
> inode.
>
> The size is stored in what used to be a 4-byte hole. If necessary the
> field can be made smaller and converted into a union with something not
> used with symlinks.
>
> Churn-wise the current readlink_copy() helper is patched to accept the
> size instead of calculating it.
>
> Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
> ---
> fs/namei.c | 34 +++++++++++++++++++---------------
> fs/proc/namespaces.c | 2 +-
> include/linux/fs.h | 12 ++++++++++--
> security/apparmor/apparmorfs.c | 2 +-
> 4 files changed, 31 insertions(+), 19 deletions(-)
>
> diff --git a/fs/namei.c b/fs/namei.c
> index 9d30c7aa9aa6..e56c29a22d26 100644
> --- a/fs/namei.c
> +++ b/fs/namei.c
> @@ -5272,19 +5272,16 @@ SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newna
> getname(newname), 0);
> }
>
> -int readlink_copy(char __user *buffer, int buflen, const char *link)
> +int readlink_copy(char __user *buffer, int buflen, const char *link, int linklen)
> {
> - int len = PTR_ERR(link);
> - if (IS_ERR(link))
> - goto out;
> + int copylen;
>
> - len = strlen(link);
> - if (len > (unsigned) buflen)
> - len = buflen;
> - if (copy_to_user(buffer, link, len))
> - len = -EFAULT;
> -out:
> - return len;
> + copylen = linklen;
> + if (unlikely(copylen > (unsigned) buflen))
> + copylen = buflen;
> + if (copy_to_user(buffer, link, copylen))
> + copylen = -EFAULT;
> + return copylen;
> }
>
> /**
> @@ -5304,6 +5301,9 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
> const char *link;
> int res;
>
> + if (inode->i_opflags & IOP_CACHED_LINK)
> + return readlink_copy(buffer, buflen, inode->i_link, inode->i_linklen);
> +
> if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
> if (unlikely(inode->i_op->readlink))
> return inode->i_op->readlink(dentry, buffer, buflen);
> @@ -5322,7 +5322,7 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
> if (IS_ERR(link))
> return PTR_ERR(link);
> }
> - res = readlink_copy(buffer, buflen, link);
> + res = readlink_copy(buffer, buflen, link, strlen(link));
> do_delayed_call(&done);
> return res;
> }
> @@ -5391,10 +5391,14 @@ EXPORT_SYMBOL(page_put_link);
>
> int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
> {
> + const char *link;
> + int res;
> +
> DEFINE_DELAYED_CALL(done);
> - int res = readlink_copy(buffer, buflen,
> - page_get_link(dentry, d_inode(dentry),
> - &done));
> + link = page_get_link(dentry, d_inode(dentry), &done);
> + res = PTR_ERR(link);
> + if (!IS_ERR(link))
> + res = readlink_copy(buffer, buflen, link, strlen(link));
> do_delayed_call(&done);
> return res;
> }
> diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c
> index 8e159fc78c0a..c610224faf10 100644
> --- a/fs/proc/namespaces.c
> +++ b/fs/proc/namespaces.c
> @@ -83,7 +83,7 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl
> if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
> res = ns_get_name(name, sizeof(name), task, ns_ops);
> if (res >= 0)
> - res = readlink_copy(buffer, buflen, name);
> + res = readlink_copy(buffer, buflen, name, strlen(name));
> }
> put_task_struct(task);
> return res;
> diff --git a/include/linux/fs.h b/include/linux/fs.h
> index 972147da71f9..30e332fb399d 100644
> --- a/include/linux/fs.h
> +++ b/include/linux/fs.h
> @@ -626,6 +626,7 @@ is_uncached_acl(struct posix_acl *acl)
> #define IOP_XATTR 0x0008
> #define IOP_DEFAULT_READLINK 0x0010
> #define IOP_MGTIME 0x0020
> +#define IOP_CACHED_LINK 0x0040
>
> /*
> * Keep mostly read-only and often accessed (especially for
> @@ -686,7 +687,7 @@ struct inode {
>
> /* Misc */
> u32 i_state;
> - /* 32-bit hole */
> + int i_linklen; /* for symlinks */
> struct rw_semaphore i_rwsem;
>
> unsigned long dirtied_when; /* jiffies of first dirtying */
> @@ -749,6 +750,13 @@ struct inode {
> void *i_private; /* fs or device private pointer */
> } __randomize_layout;
>
> +static inline void inode_set_cached_link(struct inode *inode, char *link, int linklen)
> +{
> + inode->i_link = link;
> + inode->i_linklen = linklen;
Just curious, is this linklen equal to inode size? if it is, why don't
use it?
> + inode->i_opflags |= IOP_CACHED_LINK;
> +}
> +
> /*
> * Get bit address from inode->i_state to use with wait_var_event()
> * infrastructre.
> @@ -3351,7 +3359,7 @@ extern const struct file_operations generic_ro_fops;
>
> #define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))
>
> -extern int readlink_copy(char __user *, int, const char *);
> +extern int readlink_copy(char __user *, int, const char *, int);
> extern int page_readlink(struct dentry *, char __user *, int);
> extern const char *page_get_link(struct dentry *, struct inode *,
> struct delayed_call *);
> diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c
> index 01b923d97a44..60959cfba672 100644
> --- a/security/apparmor/apparmorfs.c
> +++ b/security/apparmor/apparmorfs.c
> @@ -2611,7 +2611,7 @@ static int policy_readlink(struct dentry *dentry, char __user *buffer,
> res = snprintf(name, sizeof(name), "%s:[%lu]", AAFS_NAME,
> d_inode(dentry)->i_ino);
> if (res > 0 && res < sizeof(name))
> - res = readlink_copy(buffer, buflen, name);
> + res = readlink_copy(buffer, buflen, name, strlen(name));
> else
> res = -ENOENT;
>
--
Regards
On Wed, Nov 20, 2024 at 12:15:18PM +0800, wangjianjian (C) wrote:
> > +{
> > + inode->i_link = link;
> > + inode->i_linklen = linklen;
> Just curious, is this linklen equal to inode size? if it is, why don't use
> it?
Maybe you should read v1 of the patch series where Jan explained where
that's not true.
On 2024/11/20 13:01, Matthew Wilcox wrote:
> On Wed, Nov 20, 2024 at 12:15:18PM +0800, wangjianjian (C) wrote:
>>> +{
>>> + inode->i_link = link;
>>> + inode->i_linklen = linklen;
>> Just curious, is this linklen equal to inode size? if it is, why don't use
>> it?
>
> Maybe you should read v1 of the patch series where Jan explained where
> that's not true.
okay, I see, thanks for this.
>
--
Regards
© 2016 - 2026 Red Hat, Inc.