include/qemu/plugin.h | 12 ++++++++++++ include/qemu/qemu-plugin.h | 13 +++++++++++++ bsd-user/syscall.c | 6 +++--- linux-user/exit.c | 2 +- plugins/core.c | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 4 deletions(-)
In user-mode emulation there is a small race between preexit_cleanup
and exit_group() which means we may end up calling instrumented
instructions before the kernel reaps child threads. To solve this we
implement a new helper which ensures the callbacks are flushed along
with any translations before we let the host do it's a thing.
While we are at it make the documentation of
qemu_plugin_register_atexit_cb clearer as to what the user can expect.
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
---
include/qemu/plugin.h | 12 ++++++++++++
include/qemu/qemu-plugin.h | 13 +++++++++++++
bsd-user/syscall.c | 6 +++---
linux-user/exit.c | 2 +-
plugins/core.c | 33 +++++++++++++++++++++++++++++++++
5 files changed, 62 insertions(+), 4 deletions(-)
diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h
index 0fefbc6084..9a8438f683 100644
--- a/include/qemu/plugin.h
+++ b/include/qemu/plugin.h
@@ -190,6 +190,16 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr);
void qemu_plugin_disable_mem_helpers(CPUState *cpu);
+/**
+ * qemu_plugin_user_exit(): clean-up callbacks before calling exit callbacks
+ *
+ * This is a user-mode only helper that ensure we have fully cleared
+ * callbacks from all threads before calling the exit callbacks. This
+ * is so the plugins themselves don't have to jump through hoops to
+ * guard against race conditions.
+ */
+void qemu_plugin_user_exit(void);
+
#else /* !CONFIG_PLUGIN */
static inline void qemu_plugin_add_opts(void)
@@ -250,6 +260,8 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr)
static inline void qemu_plugin_disable_mem_helpers(CPUState *cpu)
{ }
+static inline void qemu_plugin_user_exit(void)
+{ }
#endif /* !CONFIG_PLUGIN */
#endif /* QEMU_PLUGIN_H */
diff --git a/include/qemu/qemu-plugin.h b/include/qemu/qemu-plugin.h
index dc3496f36c..e6e815abc5 100644
--- a/include/qemu/qemu-plugin.h
+++ b/include/qemu/qemu-plugin.h
@@ -549,6 +549,19 @@ void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id,
void qemu_plugin_register_flush_cb(qemu_plugin_id_t id,
qemu_plugin_simple_cb_t cb);
+/**
+ * qemu_plugin_register_atexit_cb() - register exit callback
+ * @id: plugin ID
+ * @cb: callback
+ * @userdata: user data for callback
+ *
+ * The @cb function is called once execution has finished. Plugins
+ * should be able to free all their resources at this point much like
+ * after a reset/uninstall callback is called.
+ *
+ * In user-mode it is possible a few un-instrumented instructions from
+ * child threads may run before the host kernel reaps the threads.
+ */
void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
qemu_plugin_udata_cb_t cb, void *userdata);
diff --git a/bsd-user/syscall.c b/bsd-user/syscall.c
index 7d986e9700..3f44311396 100644
--- a/bsd-user/syscall.c
+++ b/bsd-user/syscall.c
@@ -335,7 +335,7 @@ abi_long do_freebsd_syscall(void *cpu_env, int num, abi_long arg1,
_mcleanup();
#endif
gdb_exit(arg1);
- qemu_plugin_atexit_cb();
+ qemu_plugin_user_exit();
/* XXX: should free thread stack and CPU env */
_exit(arg1);
ret = 0; /* avoid warning */
@@ -437,7 +437,7 @@ abi_long do_netbsd_syscall(void *cpu_env, int num, abi_long arg1,
_mcleanup();
#endif
gdb_exit(arg1);
- qemu_plugin_atexit_cb();
+ qemu_plugin_user_exit();
/* XXX: should free thread stack and CPU env */
_exit(arg1);
ret = 0; /* avoid warning */
@@ -516,7 +516,7 @@ abi_long do_openbsd_syscall(void *cpu_env, int num, abi_long arg1,
_mcleanup();
#endif
gdb_exit(arg1);
- qemu_plugin_atexit_cb();
+ qemu_plugin_user_exit();
/* XXX: should free thread stack and CPU env */
_exit(arg1);
ret = 0; /* avoid warning */
diff --git a/linux-user/exit.c b/linux-user/exit.c
index 70b344048c..527e29cbc1 100644
--- a/linux-user/exit.c
+++ b/linux-user/exit.c
@@ -35,5 +35,5 @@ void preexit_cleanup(CPUArchState *env, int code)
__gcov_dump();
#endif
gdb_exit(code);
- qemu_plugin_atexit_cb();
+ qemu_plugin_user_exit();
}
diff --git a/plugins/core.c b/plugins/core.c
index e1bcdb570d..c573b81a96 100644
--- a/plugins/core.c
+++ b/plugins/core.c
@@ -487,6 +487,39 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
plugin_register_cb_udata(id, QEMU_PLUGIN_EV_ATEXIT, cb, udata);
}
+/*
+ * Handle exit from linux-user. Unlike the normal atexit() mechanism
+ * we need to handle the clean-up manually as it's possible threads
+ * are still running. We need to remove all callbacks from code
+ * generation, flush the current translations and then we can safely
+ * trigger the exit callbacks.
+ */
+
+void qemu_plugin_user_exit(void)
+{
+ enum qemu_plugin_event ev;
+
+ QEMU_LOCK_GUARD(&plugin.lock);
+
+ start_exclusive();
+
+ /* un-register all callbacks except the final AT_EXIT one */
+ for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
+ if (ev != QEMU_PLUGIN_EV_ATEXIT) {
+ struct qemu_plugin_ctx *ctx;
+ QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
+ plugin_unregister_cb__locked(ctx, ev);
+ }
+ }
+ }
+
+ tb_flush(current_cpu);
+ end_exclusive();
+
+ /* now it's safe to handle the exit case */
+ qemu_plugin_atexit_cb();
+}
+
/*
* Call this function after longjmp'ing to the main loop. It's possible that the
* last instruction of a TB might have used helpers, and therefore the
--
2.32.0.264.g75ae10bc75
Alex Bennée <alex.bennee@linaro.org> writes: > In user-mode emulation there is a small race between preexit_cleanup > and exit_group() which means we may end up calling instrumented > instructions before the kernel reaps child threads. To solve this we > implement a new helper which ensures the callbacks are flushed along > with any translations before we let the host do it's a thing. > > While we are at it make the documentation of > qemu_plugin_register_atexit_cb clearer as to what the user can expect. > <snip> > > +/* > + * Handle exit from linux-user. Unlike the normal atexit() mechanism > + * we need to handle the clean-up manually as it's possible threads > + * are still running. We need to remove all callbacks from code > + * generation, flush the current translations and then we can safely > + * trigger the exit callbacks. > + */ > + > +void qemu_plugin_user_exit(void) > +{ > + enum qemu_plugin_event ev; > + > + QEMU_LOCK_GUARD(&plugin.lock); > + > + start_exclusive(); > + > + /* un-register all callbacks except the final AT_EXIT one */ > + for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) { > + if (ev != QEMU_PLUGIN_EV_ATEXIT) { > + struct qemu_plugin_ctx *ctx; > + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { > + plugin_unregister_cb__locked(ctx, ev); > + } > + } > + } > + > + tb_flush(current_cpu); We also need to disable memory helpers during the exclusive period as that is another route into a callback: --8<---------------cut here---------------start------------->8--- modified plugins/core.c @@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, void qemu_plugin_user_exit(void) { enum qemu_plugin_event ev; + CPUState *cpu; QEMU_LOCK_GUARD(&plugin.lock); @@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void) } tb_flush(current_cpu); + + CPU_FOREACH(cpu) { + qemu_plugin_disable_mem_helpers(cpu); + } + end_exclusive(); /* now it's safe to handle the exit case */ --8<---------------cut here---------------end--------------->8--- > + end_exclusive(); > + > + /* now it's safe to handle the exit case */ > + qemu_plugin_atexit_cb(); > +} > + > /* > * Call this function after longjmp'ing to the main loop. It's possible that the > * last instruction of a TB might have used helpers, and therefore the -- Alex Bennée
On Mon, Jul 19, 2021, 7:57 AM Alex Bennée <alex.bennee@linaro.org> wrote: > > Alex Bennée <alex.bennee@linaro.org> writes: > > > In user-mode emulation there is a small race between preexit_cleanup > > and exit_group() which means we may end up calling instrumented > > instructions before the kernel reaps child threads. To solve this we > > implement a new helper which ensures the callbacks are flushed along > > with any translations before we let the host do it's a thing. > > > > While we are at it make the documentation of > > qemu_plugin_register_atexit_cb clearer as to what the user can expect. > > > <snip> > > > > +/* > > + * Handle exit from linux-user. Unlike the normal atexit() mechanism > > + * we need to handle the clean-up manually as it's possible threads > > + * are still running. We need to remove all callbacks from code > > + * generation, flush the current translations and then we can safely > > + * trigger the exit callbacks. > > + */ > > + > > +void qemu_plugin_user_exit(void) > > +{ > > + enum qemu_plugin_event ev; > > + > > + QEMU_LOCK_GUARD(&plugin.lock); > > + > > + start_exclusive(); > > + > > + /* un-register all callbacks except the final AT_EXIT one */ > > + for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) { > > + if (ev != QEMU_PLUGIN_EV_ATEXIT) { > > + struct qemu_plugin_ctx *ctx; > > + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { > > + plugin_unregister_cb__locked(ctx, ev); > > + } > > + } > > + } > > + > > + tb_flush(current_cpu); > > We also need to disable memory helpers during the exclusive period as > that is another route into a callback: > --8<---------------cut here---------------start------------->8--- > modified plugins/core.c > @@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t > id, > void qemu_plugin_user_exit(void) > { > enum qemu_plugin_event ev; > + CPUState *cpu; > > QEMU_LOCK_GUARD(&plugin.lock); > > @@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void) > } > > tb_flush(current_cpu); > + > + CPU_FOREACH(cpu) { > + qemu_plugin_disable_mem_helpers(cpu); > + } > + > end_exclusive(); > > /* now it's safe to handle the exit case */ > --8<---------------cut here---------------end--------------->8--- > I think both of these are find from a bsd-user point of view. Warner > > + end_exclusive(); > > + > > + /* now it's safe to handle the exit case */ > > + qemu_plugin_atexit_cb(); > > +} > > + > > /* > > * Call this function after longjmp'ing to the main loop. It's possible > that the > > * last instruction of a TB might have used helpers, and therefore the > > > -- > Alex Bennée >
Warner Losh <imp@bsdimp.com> writes: > On Mon, Jul 19, 2021, 7:57 AM Alex Bennée <alex.bennee@linaro.org> wrote: > > Alex Bennée <alex.bennee@linaro.org> writes: > > > In user-mode emulation there is a small race between preexit_cleanup > > and exit_group() which means we may end up calling instrumented > > instructions before the kernel reaps child threads. To solve this we > > implement a new helper which ensures the callbacks are flushed along > > with any translations before we let the host do it's a thing. > > > > While we are at it make the documentation of > > qemu_plugin_register_atexit_cb clearer as to what the user can expect. > > > <snip> > > > > +/* > > + * Handle exit from linux-user. Unlike the normal atexit() mechanism > > + * we need to handle the clean-up manually as it's possible threads > > + * are still running. We need to remove all callbacks from code > > + * generation, flush the current translations and then we can safely > > + * trigger the exit callbacks. > > + */ > > + > > +void qemu_plugin_user_exit(void) > > +{ > > + enum qemu_plugin_event ev; > > + > > + QEMU_LOCK_GUARD(&plugin.lock); > > + > > + start_exclusive(); > > + > > + /* un-register all callbacks except the final AT_EXIT one */ > > + for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) { > > + if (ev != QEMU_PLUGIN_EV_ATEXIT) { > > + struct qemu_plugin_ctx *ctx; > > + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { > > + plugin_unregister_cb__locked(ctx, ev); > > + } > > + } > > + } > > + > > + tb_flush(current_cpu); > > We also need to disable memory helpers during the exclusive period as > that is another route into a callback: > --8<---------------cut here---------------start------------->8--- > modified plugins/core.c > @@ -498,6 +499,7 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, > void qemu_plugin_user_exit(void) > { > enum qemu_plugin_event ev; > + CPUState *cpu; > > QEMU_LOCK_GUARD(&plugin.lock); > > @@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void) > } > > tb_flush(current_cpu); > + > + CPU_FOREACH(cpu) { > + qemu_plugin_disable_mem_helpers(cpu); > + } > + > end_exclusive(); > > /* now it's safe to handle the exit case */ > --8<---------------cut here---------------end--------------->8--- > > I think both of these are find from a bsd-user point of view. Acked-by: or Reviewed-by:? -- Alex Bennée
On Mon, Jul 19, 2021, 1:22 PM Alex Bennée <alex.bennee@linaro.org> wrote: > > Warner Losh <imp@bsdimp.com> writes: > > > On Mon, Jul 19, 2021, 7:57 AM Alex Bennée <alex.bennee@linaro.org> > wrote: > > > > Alex Bennée <alex.bennee@linaro.org> writes: > > > > > In user-mode emulation there is a small race between preexit_cleanup > > > and exit_group() which means we may end up calling instrumented > > > instructions before the kernel reaps child threads. To solve this we > > > implement a new helper which ensures the callbacks are flushed along > > > with any translations before we let the host do it's a thing. > > > > > > While we are at it make the documentation of > > > qemu_plugin_register_atexit_cb clearer as to what the user can expect. > > > > > <snip> > > > > > > +/* > > > + * Handle exit from linux-user. Unlike the normal atexit() mechanism > > > + * we need to handle the clean-up manually as it's possible threads > > > + * are still running. We need to remove all callbacks from code > > > + * generation, flush the current translations and then we can safely > > > + * trigger the exit callbacks. > > > + */ > > > + > > > +void qemu_plugin_user_exit(void) > > > +{ > > > + enum qemu_plugin_event ev; > > > + > > > + QEMU_LOCK_GUARD(&plugin.lock); > > > + > > > + start_exclusive(); > > > + > > > + /* un-register all callbacks except the final AT_EXIT one */ > > > + for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) { > > > + if (ev != QEMU_PLUGIN_EV_ATEXIT) { > > > + struct qemu_plugin_ctx *ctx; > > > + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { > > > + plugin_unregister_cb__locked(ctx, ev); > > > + } > > > + } > > > + } > > > + > > > + tb_flush(current_cpu); > > > > We also need to disable memory helpers during the exclusive period as > > that is another route into a callback: > > --8<---------------cut here---------------start------------->8--- > > modified plugins/core.c > > @@ -498,6 +499,7 @@ void > qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, > > void qemu_plugin_user_exit(void) > > { > > enum qemu_plugin_event ev; > > + CPUState *cpu; > > > > QEMU_LOCK_GUARD(&plugin.lock); > > > > @@ -514,6 +516,11 @@ void qemu_plugin_user_exit(void) > > } > > > > tb_flush(current_cpu); > > + > > + CPU_FOREACH(cpu) { > > + qemu_plugin_disable_mem_helpers(cpu); > > + } > > + > > end_exclusive(); > > > > /* now it's safe to handle the exit case */ > > --8<---------------cut here---------------end--------------->8--- > > > > I think both of these are find from a bsd-user point of view. > > Acked-by: or Reviewed-by:? > Sorry I wasn't clear. Acked-by: Warner Losh <imp@bsdimp.com> > -- > Alex Bennée >
On Mon, Jul 19, 2021, 14:37 Alex Bennée <alex.bennee@linaro.org> wrote: > In user-mode emulation there is a small race between preexit_cleanup > and exit_group() which means we may end up calling instrumented > instructions before the kernel reaps child threads. To solve this we > implement a new helper which ensures the callbacks are flushed along > with any translations before we let the host do it's a thing. > > While we are at it make the documentation of > qemu_plugin_register_atexit_cb clearer as to what the user can expect. > > Signed-off-by: Alex Bennée <alex.bennee@linaro.org> > Looks great to me and I cannot reproduce the race with it installed in. With calling `qemu_plugin_disable_mem_helpers`: Reviewed-by: Mahmoud Mandour <ma.mandourr@gmail.com> --- > include/qemu/plugin.h | 12 ++++++++++++ > include/qemu/qemu-plugin.h | 13 +++++++++++++ > bsd-user/syscall.c | 6 +++--- > linux-user/exit.c | 2 +- > plugins/core.c | 33 +++++++++++++++++++++++++++++++++ > 5 files changed, 62 insertions(+), 4 deletions(-) > > diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h > index 0fefbc6084..9a8438f683 100644 > --- a/include/qemu/plugin.h > +++ b/include/qemu/plugin.h > @@ -190,6 +190,16 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr); > > void qemu_plugin_disable_mem_helpers(CPUState *cpu); > > +/** > + * qemu_plugin_user_exit(): clean-up callbacks before calling exit > callbacks > + * > + * This is a user-mode only helper that ensure we have fully cleared > + * callbacks from all threads before calling the exit callbacks. This > + * is so the plugins themselves don't have to jump through hoops to > + * guard against race conditions. > + */ > +void qemu_plugin_user_exit(void); > + > #else /* !CONFIG_PLUGIN */ > > static inline void qemu_plugin_add_opts(void) > @@ -250,6 +260,8 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr) > static inline void qemu_plugin_disable_mem_helpers(CPUState *cpu) > { } > > +static inline void qemu_plugin_user_exit(void) > +{ } > #endif /* !CONFIG_PLUGIN */ > > #endif /* QEMU_PLUGIN_H */ > diff --git a/include/qemu/qemu-plugin.h b/include/qemu/qemu-plugin.h > index dc3496f36c..e6e815abc5 100644 > --- a/include/qemu/qemu-plugin.h > +++ b/include/qemu/qemu-plugin.h > @@ -549,6 +549,19 @@ void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id, > void qemu_plugin_register_flush_cb(qemu_plugin_id_t id, > qemu_plugin_simple_cb_t cb); > > +/** > + * qemu_plugin_register_atexit_cb() - register exit callback > + * @id: plugin ID > + * @cb: callback > + * @userdata: user data for callback > + * > + * The @cb function is called once execution has finished. Plugins > + * should be able to free all their resources at this point much like > + * after a reset/uninstall callback is called. > + * > + * In user-mode it is possible a few un-instrumented instructions from > + * child threads may run before the host kernel reaps the threads. > + */ > void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id, > qemu_plugin_udata_cb_t cb, void > *userdata); > > diff --git a/bsd-user/syscall.c b/bsd-user/syscall.c > index 7d986e9700..3f44311396 100644 > --- a/bsd-user/syscall.c > +++ b/bsd-user/syscall.c > @@ -335,7 +335,7 @@ abi_long do_freebsd_syscall(void *cpu_env, int num, > abi_long arg1, > _mcleanup(); > #endif > gdb_exit(arg1); > - qemu_plugin_atexit_cb(); > + qemu_plugin_user_exit(); > /* XXX: should free thread stack and CPU env */ > _exit(arg1); > ret = 0; /* avoid warning */ > @@ -437,7 +437,7 @@ abi_long do_netbsd_syscall(void *cpu_env, int num, > abi_long arg1, > _mcleanup(); > #endif > gdb_exit(arg1); > - qemu_plugin_atexit_cb(); > + qemu_plugin_user_exit(); > /* XXX: should free thread stack and CPU env */ > _exit(arg1); > ret = 0; /* avoid warning */ > @@ -516,7 +516,7 @@ abi_long do_openbsd_syscall(void *cpu_env, int num, > abi_long arg1, > _mcleanup(); > #endif > gdb_exit(arg1); > - qemu_plugin_atexit_cb(); > + qemu_plugin_user_exit(); > /* XXX: should free thread stack and CPU env */ > _exit(arg1); > ret = 0; /* avoid warning */ > diff --git a/linux-user/exit.c b/linux-user/exit.c > index 70b344048c..527e29cbc1 100644 > --- a/linux-user/exit.c > +++ b/linux-user/exit.c > @@ -35,5 +35,5 @@ void preexit_cleanup(CPUArchState *env, int code) > __gcov_dump(); > #endif > gdb_exit(code); > - qemu_plugin_atexit_cb(); > + qemu_plugin_user_exit(); > } > diff --git a/plugins/core.c b/plugins/core.c > index e1bcdb570d..c573b81a96 100644 > --- a/plugins/core.c > +++ b/plugins/core.c > @@ -487,6 +487,39 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t > id, > plugin_register_cb_udata(id, QEMU_PLUGIN_EV_ATEXIT, cb, udata); > } > > +/* > + * Handle exit from linux-user. Unlike the normal atexit() mechanism > + * we need to handle the clean-up manually as it's possible threads > + * are still running. We need to remove all callbacks from code > + * generation, flush the current translations and then we can safely > + * trigger the exit callbacks. > + */ > + > +void qemu_plugin_user_exit(void) > +{ > + enum qemu_plugin_event ev; > + > + QEMU_LOCK_GUARD(&plugin.lock); > + > + start_exclusive(); > + > + /* un-register all callbacks except the final AT_EXIT one */ > + for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) { > + if (ev != QEMU_PLUGIN_EV_ATEXIT) { > + struct qemu_plugin_ctx *ctx; > + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { > + plugin_unregister_cb__locked(ctx, ev); > + } > + } > + } > + > + tb_flush(current_cpu); > + end_exclusive(); > + > + /* now it's safe to handle the exit case */ > + qemu_plugin_atexit_cb(); > +} > + > /* > * Call this function after longjmp'ing to the main loop. It's possible > that the > * last instruction of a TB might have used helpers, and therefore the > -- > 2.32.0.264.g75ae10bc75 > >
© 2016 - 2024 Red Hat, Inc.