This commit implements the -native-bypass support in linux-user. The
native_calls_enabled() function can be true only when the
'-native-bypass' option is given.

Signed-off-by: Yeqi Fu <fufuyqqqqqq@gmail.com>
---
 include/native/native.h |  9 +++++++++
 linux-user/main.c       | 38 ++++++++++++++++++++++++++++++++++++++
 linux-user/syscall.c    | 21 +++++++++++++++++++++
 3 files changed, 68 insertions(+)
 create mode 100644 include/native/native.h

diff --git a/include/native/native.h b/include/native/native.h
new file mode 100644
index 0000000000..7d1baadfcf
--- /dev/null
+++ b/include/native/native.h
@@ -0,0 +1,9 @@
+/*
+ * Check if the native bypass feature is enabled.
+ */
+#if defined(CONFIG_USER_ONLY) && defined(CONFIG_NATIVE_CALL)
+extern char *native_lib_path;
+#define native_bypass_enabled() (native_lib_path != NULL)
+#else
+#define native_bypass_enabled() false
+#endif
diff --git a/linux-user/main.c b/linux-user/main.c
index dba67ffa36..5cf02c071b 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -60,6 +60,11 @@
 #include "semihosting/semihost.h"
 #endif
 
+#if defined(CONFIG_NATIVE_CALL)
+#include "native/native.h"
+char *native_lib_path;
+#endif
+
 #ifndef AT_FLAGS_PRESERVE_ARGV0
 #define AT_FLAGS_PRESERVE_ARGV0_BIT 0
 #define AT_FLAGS_PRESERVE_ARGV0 (1 << AT_FLAGS_PRESERVE_ARGV0_BIT)
@@ -293,6 +298,17 @@ static void handle_arg_set_env(const char *arg)
     free(r);
 }
 
+#if defined(CONFIG_NATIVE_CALL)
+static void handle_arg_native_bypass(const char *arg)
+{
+    if (access(arg, F_OK) != 0) {
+        fprintf(stderr, "native library %s does not exist\n", arg);
+        exit(EXIT_FAILURE);
+    }
+    native_lib_path = g_strdup(arg);
+}
+#endif
+
 static void handle_arg_unset_env(const char *arg)
 {
     char *r, *p, *token;
@@ -522,6 +538,10 @@ static const struct qemu_argument arg_table[] = {
      "",           "Generate a /tmp/perf-${pid}.map file for perf"},
     {"jitdump",    "QEMU_JITDUMP",     false, handle_arg_jitdump,
      "",           "Generate a jit-${pid}.dump file for perf"},
+#if defined(CONFIG_NATIVE_CALL)
+    {"native-bypass", "QEMU_NATIVE_BYPASS", true, handle_arg_native_bypass,
+     "",           "native bypass for library calls"},
+#endif
     {NULL, NULL, false, NULL, NULL, NULL}
 };
 
@@ -834,6 +854,24 @@ int main(int argc, char **argv, char **envp)
         }
     }
 
+#if defined(CONFIG_NATIVE_CALL)
+    /* Set the library for native bypass  */
+    if (native_lib_path) {
+        if (g_file_test(native_lib_path, G_FILE_TEST_IS_REGULAR)) {
+            GString *lib = g_string_new(native_lib_path);
+            lib = g_string_prepend(lib, "LD_PRELOAD=");
+            if (envlist_appendenv(envlist, g_string_free(lib, false), ":")) {
+                fprintf(stderr,
+                    "failed to append the native library to environment.\n");
+                exit(EXIT_FAILURE);
+            }
+        } else {
+            fprintf(stderr, "native library %s does not exist.\n",
+                    native_lib_path);
+            exit(EXIT_FAILURE);
+        }
+    }
+#endif
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index 08162cc966..bd4c3045ff 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -143,6 +143,7 @@
 #include "fd-trans.h"
 #include "tcg/tcg.h"
 #include "cpu_loop-common.h"
+#include "native/native.h"
 
 #ifndef CLONE_IO
 #define CLONE_IO                0x80000000      /* Clone io context */
@@ -8626,6 +8627,7 @@ static int do_execveat(CPUArchState *cpu_env, int dirfd,
     abi_ulong addr;
     char **q;
     void *p;
+    unsigned int i;
 
     argc = 0;
 
@@ -8696,6 +8698,25 @@ static int do_execveat(CPUArchState *cpu_env, int dirfd,
         goto execve_efault;
     }
 
+    /*
+     * An error may occur when executing execv, stating that the
+     * shared library from LD_PRELOAD cannot be preloaded on a
+     * different arch. So, we find LD_PRELOAD and remove it from
+     * envp before executing the execv.
+     */
+    if (native_bypass_enabled()) {
+        i = 0;
+        while (envp[i] != NULL) {
+            if (strncmp(envp[i], "LD_PRELOAD=", 11) == 0) {
+                for (int j = i; envp[j] != NULL; j++) {
+                    envp[j] = envp[j + 1];
+                }
+            } else {
+                i++;
+            }
+        }
+    }
+
     if (is_proc_myself(p, "exe")) {
         ret = get_errno(safe_execveat(dirfd, exec_path, argp, envp, flags));
     } else {
-- 
2.34.1

On 8/25/23 03:20, Yeqi Fu wrote:
> +#if defined(CONFIG_NATIVE_CALL)
> +    /* Set the library for native bypass  */
> +    if (native_lib_path) {
> +        if (g_file_test(native_lib_path, G_FILE_TEST_IS_REGULAR)) {
> +            GString *lib = g_string_new(native_lib_path);
> +            lib = g_string_prepend(lib, "LD_PRELOAD=");
> +            if (envlist_appendenv(envlist, g_string_free(lib, false), ":")) {
> +                fprintf(stderr,
> +                    "failed to append the native library to environment.\n");
> +                exit(EXIT_FAILURE);
> +            }
> +        } else {
> +            fprintf(stderr, "native library %s does not exist.\n",
> +                    native_lib_path);
> +            exit(EXIT_FAILURE);
> +        }
> +    }
> +#endif

Here you append to the existing LD_PRELOAD.

> +    /*
> +     * An error may occur when executing execv, stating that the
> +     * shared library from LD_PRELOAD cannot be preloaded on a
> +     * different arch. So, we find LD_PRELOAD and remove it from
> +     * envp before executing the execv.
> +     */
> +    if (native_bypass_enabled()) {
> +        i = 0;
> +        while (envp[i] != NULL) {
> +            if (strncmp(envp[i], "LD_PRELOAD=", 11) == 0) {
> +                for (int j = i; envp[j] != NULL; j++) {
> +                    envp[j] = envp[j + 1];
> +                }
> +            } else {
> +                i++;
> +            }
> +        }
> +    }

Here you simply remove LD_PRELOAD entirely.
At most you should only remove libnative.so.

I'm not at all sure that you should be modifying the target environment at all.  It's ok 
for simple testing, but it is definitely error prone.  There are a couple of different 
solutions:

(1) Dynamically modify /etc/ld.so.preload, similar to how we handle various /proc files.

(2) Merge libnative.so with vdso.so (and select one of two images depending on bypass 
enabled).


r~