From: Ilya Leoshkevich <iii@linux.ibm.com>

Add libdw-based functions for loading and querying debuginfo. Load

debuginfo from the system and the linux-user loaders.

This is useful for the upcoming perf support, which can then put

human-readable guest symbols instead of raw guest PCs into perfmap and

jitdump files.

Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>

Message-Id: <20230112152013.125680-3-iii@linux.ibm.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

meson.build | 8 ++++

accel/tcg/debuginfo.h | 77 +++++++++++++++++++++++++++++++++

accel/tcg/debuginfo.c | 96 ++++++++++++++++++++++++++++++++++++++++++

hw/core/loader.c | 5 +++

linux-user/elfload.c | 3 ++

accel/tcg/meson.build | 1 +

linux-user/meson.build | 1 +

7 files changed, 191 insertions(+)

create mode 100644 accel/tcg/debuginfo.h

create mode 100644 accel/tcg/debuginfo.c

diff --git a/meson.build b/meson.build

index XXXXXXX..XXXXXXX 100644

--- a/meson.build

+++ b/meson.build

@@ -XXX,XX +XXX,XX @@ if libbpf.found() and not cc.links('''

endif

endif

+# libdw

+libdw = dependency('libdw',

+ method: 'pkg-config',

+ kwargs: static_kwargs,

+ required: false)

+

#################

# config-host.h #

#################

@@ -XXX,XX +XXX,XX @@ config_host_data.set('CONFIG_DBUS_DISPLAY', dbus_display)

config_host_data.set('CONFIG_CFI', get_option('cfi'))

config_host_data.set('CONFIG_SELINUX', selinux.found())

config_host_data.set('CONFIG_XEN_BACKEND', xen.found())

+config_host_data.set('CONFIG_LIBDW', libdw.found())

if xen.found()

# protect from xen.version() having less than three components

xen_version = xen.version().split('.') + ['0', '0']

@@ -XXX,XX +XXX,XX @@ summary_info += {'libudev': libudev}

# Dummy dependency, keep .found()

summary_info += {'FUSE lseek': fuse_lseek.found()}

summary_info += {'selinux': selinux}

+summary_info += {'libdw': libdw}

summary(summary_info, bool_yn: true, section: 'Dependencies')

if not supported_cpus.contains(cpu)

diff --git a/accel/tcg/debuginfo.h b/accel/tcg/debuginfo.h

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/accel/tcg/debuginfo.h

@@ -XXX,XX +XXX,XX @@

+/*

+ * Debug information support.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ */

+

+#ifndef ACCEL_TCG_DEBUGINFO_H

+#define ACCEL_TCG_DEBUGINFO_H

+

+/*

+ * Debuginfo describing a certain address.

+ */

+struct debuginfo_query {

+ uint64_t address; /* Input: address. */

+ int flags; /* Input: debuginfo subset. */

+ const char *symbol; /* Symbol that the address is part of. */

+ uint64_t offset; /* Offset from the symbol. */

+ const char *file; /* Source file associated with the address. */

+ int line; /* Line number in the source file. */

+};

+

+/*

+ * Debuginfo subsets.

+ */

+#define DEBUGINFO_SYMBOL BIT(1)

+#define DEBUGINFO_LINE BIT(2)

+

+#if defined(CONFIG_TCG) && defined(CONFIG_LIBDW)

+/*

+ * Load debuginfo for the specified guest ELF image.

+ * Return true on success, false on failure.

+ */

+void debuginfo_report_elf(const char *name, int fd, uint64_t bias);

+

+/*

+ * Take the debuginfo lock.

+ */

+void debuginfo_lock(void);

+

+/*

+ * Fill each on N Qs with the debuginfo about Q->ADDRESS as specified by

+ * Q->FLAGS:

+ *

+ * - DEBUGINFO_SYMBOL: update Q->SYMBOL and Q->OFFSET. If symbol debuginfo is

+ * missing, then leave them as is.

+ * - DEBUINFO_LINE: update Q->FILE and Q->LINE. If line debuginfo is missing,

+ * then leave them as is.

+ *

+ * This function must be called under the debuginfo lock. The results can be

+ * accessed only until the debuginfo lock is released.

+ */

+void debuginfo_query(struct debuginfo_query *q, size_t n);

+

+/*

+ * Release the debuginfo lock.

+ */

+void debuginfo_unlock(void);

+#else

+static inline void debuginfo_report_elf(const char *image_name, int image_fd,

+ uint64_t load_bias)

+{

+}

+

+static inline void debuginfo_lock(void)

+{

+}

+

+static inline void debuginfo_query(struct debuginfo_query *q, size_t n)

+{

+}

+

+static inline void debuginfo_unlock(void)

+{

+}

+#endif

+

+#endif

diff --git a/accel/tcg/debuginfo.c b/accel/tcg/debuginfo.c

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/accel/tcg/debuginfo.c

@@ -XXX,XX +XXX,XX @@

+/*

+ * Debug information support.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ */

+

+#include "qemu/osdep.h"

+#include "qemu/lockable.h"

+

+#include <elfutils/libdwfl.h>

+

+#include "debuginfo.h"

+

+static QemuMutex lock;

+static Dwfl *dwfl;

+static const Dwfl_Callbacks dwfl_callbacks = {

+ .find_elf = NULL,

+ .find_debuginfo = dwfl_standard_find_debuginfo,

+ .section_address = NULL,

+ .debuginfo_path = NULL,

+};

+

+__attribute__((constructor))

+static void debuginfo_init(void)

+{

+ qemu_mutex_init(&lock);

+}

+

+void debuginfo_report_elf(const char *name, int fd, uint64_t bias)

+{

+ QEMU_LOCK_GUARD(&lock);

+

+ if (dwfl) {

+ dwfl_report_begin_add(dwfl);

+ } else {

+ dwfl = dwfl_begin(&dwfl_callbacks);

+ }

+

+ if (dwfl) {

+ dwfl_report_elf(dwfl, name, name, fd, bias, true);

+ dwfl_report_end(dwfl, NULL, NULL);

+ }

+}

+

+void debuginfo_lock(void)

+{

+ qemu_mutex_lock(&lock);

+}

+

+void debuginfo_query(struct debuginfo_query *q, size_t n)

+{

+ const char *symbol, *file;

+ Dwfl_Module *dwfl_module;

+ Dwfl_Line *dwfl_line;

+ GElf_Off dwfl_offset;

+ GElf_Sym dwfl_sym;

+ size_t i;

+ int line;

+

+ if (!dwfl) {

+ return;

+ }

+

+ for (i = 0; i < n; i++) {

+ dwfl_module = dwfl_addrmodule(dwfl, q[i].address);

+ if (!dwfl_module) {

+ continue;

+ }

+

+ if (q[i].flags & DEBUGINFO_SYMBOL) {

+ symbol = dwfl_module_addrinfo(dwfl_module, q[i].address,

+ &dwfl_offset, &dwfl_sym,

+ NULL, NULL, NULL);

+ if (symbol) {

+ q[i].symbol = symbol;

+ q[i].offset = dwfl_offset;

+ }

+ }

+

+ if (q[i].flags & DEBUGINFO_LINE) {

+ dwfl_line = dwfl_module_getsrc(dwfl_module, q[i].address);

+ if (dwfl_line) {

+ file = dwfl_lineinfo(dwfl_line, NULL, &line, 0, NULL, NULL);

+ if (file) {

+ q[i].file = file;

+ q[i].line = line;

+ }

+ }

+ }

+ }

+}

+

+void debuginfo_unlock(void)

+{

+ qemu_mutex_unlock(&lock);

+}

diff --git a/hw/core/loader.c b/hw/core/loader.c

index XXXXXXX..XXXXXXX 100644

--- a/hw/core/loader.c

+++ b/hw/core/loader.c

@@ -XXX,XX +XXX,XX @@

#include "hw/boards.h"

#include "qemu/cutils.h"

#include "sysemu/runstate.h"

+#include "accel/tcg/debuginfo.h"

#include <zlib.h>

@@ -XXX,XX +XXX,XX @@ ssize_t load_elf_ram_sym(const char *filename,

clear_lsb, data_swab, as, load_rom, sym_cb);

}

+ if (ret != ELF_LOAD_FAILED) {

+ debuginfo_report_elf(filename, fd, 0);

+ }

+

fail:

close(fd);

return ret;

diff --git a/linux-user/elfload.c b/linux-user/elfload.c

index XXXXXXX..XXXXXXX 100644

--- a/linux-user/elfload.c

+++ b/linux-user/elfload.c

@@ -XXX,XX +XXX,XX @@

#include "qemu/selfmap.h"

#include "qapi/error.h"

#include "target_signal.h"

+#include "accel/tcg/debuginfo.h"

#ifdef _ARCH_PPC64

#undef ARCH_DLINFO

@@ -XXX,XX +XXX,XX @@ static void load_elf_image(const char *image_name, int image_fd,

load_symbols(ehdr, image_fd, load_bias);

}

+ debuginfo_report_elf(image_name, image_fd, load_bias);

+

mmap_unlock();

close(image_fd);

diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build

index XXXXXXX..XXXXXXX 100644

--- a/accel/tcg/meson.build

+++ b/accel/tcg/meson.build

@@ -XXX,XX +XXX,XX @@ tcg_ss.add(files(

tcg_ss.add(when: 'CONFIG_USER_ONLY', if_true: files('user-exec.c'))

tcg_ss.add(when: 'CONFIG_SOFTMMU', if_false: files('user-exec-stub.c'))

tcg_ss.add(when: 'CONFIG_PLUGIN', if_true: [files('plugin-gen.c')])

+tcg_ss.add(when: libdw, if_true: files('debuginfo.c'))

specific_ss.add_all(when: 'CONFIG_TCG', if_true: tcg_ss)

specific_ss.add(when: ['CONFIG_SOFTMMU', 'CONFIG_TCG'], if_true: files(

diff --git a/linux-user/meson.build b/linux-user/meson.build

index XXXXXXX..XXXXXXX 100644

--- a/linux-user/meson.build

+++ b/linux-user/meson.build

@@ -XXX,XX +XXX,XX @@ linux_user_ss.add(files(

'uname.c',

))

linux_user_ss.add(rt)

+linux_user_ss.add(libdw)

linux_user_ss.add(when: 'TARGET_HAS_BFLT', if_true: files('flatload.c'))

linux_user_ss.add(when: 'TARGET_I386', if_true: files('vm86.c'))

--

2.34.1

From: Ilya Leoshkevich <iii@linux.ibm.com>

Add ability to dump /tmp/perf-<pid>.map and jit-<pid>.dump.

The first one allows the perf tool to map samples to each individual

translation block. The second one adds the ability to resolve symbol

names, line numbers and inspect JITed code.

Example of use:

perf record qemu-x86_64 -perfmap ./a.out

perf report

or

perf record -k 1 qemu-x86_64 -jitdump ./a.out

DEBUGINFOD_URLS= perf inject -j -i perf.data -o perf.data.jitted

perf report -i perf.data.jitted

Co-developed-by: Vanderson M. do Rosario <vandersonmr2@gmail.com>

Co-developed-by: Alex Bennée <alex.bennee@linaro.org>

Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>

Message-Id: <20230112152013.125680-4-iii@linux.ibm.com>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---

docs/devel/tcg.rst | 23 +++

accel/tcg/perf.h | 49 +++++

accel/tcg/perf.c | 375 ++++++++++++++++++++++++++++++++++++++

accel/tcg/translate-all.c | 7 +

linux-user/exit.c | 2 +

linux-user/main.c | 15 ++

softmmu/vl.c | 11 ++

tcg/tcg.c | 2 +

accel/tcg/meson.build | 1 +

qemu-options.hx | 20 ++

10 files changed, 505 insertions(+)

create mode 100644 accel/tcg/perf.h

create mode 100644 accel/tcg/perf.c

diff --git a/docs/devel/tcg.rst b/docs/devel/tcg.rst

index XXXXXXX..XXXXXXX 100644

--- a/docs/devel/tcg.rst

+++ b/docs/devel/tcg.rst

@@ -XXX,XX +XXX,XX @@ memory areas instead calls out to C code for device emulation.

Finally, the MMU helps tracking dirty pages and pages pointed to by

translation blocks.

+Profiling JITted code

+---------------------

+

+The Linux ``perf`` tool will treat all JITted code as a single block as

+unlike the main code it can't use debug information to link individual

+program counter samples with larger functions. To overcome this

+limitation you can use the ``-perfmap`` or the ``-jitdump`` option to generate

+map files. ``-perfmap`` is lightweight and produces only guest-host mappings.

+``-jitdump`` additionally saves JITed code and guest debug information (if

+available); its output needs to be integrated with the ``perf.data`` file

+before the final report can be viewed.

+

+.. code::

+

+ perf record $QEMU -perfmap $REMAINING_ARGS

+ perf report

+

+ perf record -k 1 $QEMU -jitdump $REMAINING_ARGS

+ DEBUGINFOD_URLS= perf inject -j -i perf.data -o perf.data.jitted

+ perf report -i perf.data.jitted

+

+Note that qemu-system generates mappings only for ``-kernel`` files in ELF

+format.

diff --git a/accel/tcg/perf.h b/accel/tcg/perf.h

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/accel/tcg/perf.h

@@ -XXX,XX +XXX,XX @@

+/*

+ * Linux perf perf-<pid>.map and jit-<pid>.dump integration.

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ */

+

+#ifndef ACCEL_TCG_PERF_H

+#define ACCEL_TCG_PERF_H

+

+#if defined(CONFIG_TCG) && defined(CONFIG_LINUX)

+/* Start writing perf-<pid>.map. */

+void perf_enable_perfmap(void);

+

+/* Start writing jit-<pid>.dump. */

+void perf_enable_jitdump(void);

+

+/* Add information about TCG prologue to profiler maps. */

+void perf_report_prologue(const void *start, size_t size);

+

+/* Add information about JITted guest code to profiler maps. */

+void perf_report_code(uint64_t guest_pc, TranslationBlock *tb,

+ const void *start);

+

+/* Stop writing perf-<pid>.map and/or jit-<pid>.dump. */

+void perf_exit(void);

+#else

+static inline void perf_enable_perfmap(void)

+{

+}

+

+static inline void perf_enable_jitdump(void)

+{

+}

+

+static inline void perf_report_prologue(const void *start, size_t size)

+{

+}

+

+static inline void perf_report_code(uint64_t guest_pc, TranslationBlock *tb,

+ const void *start)

+{

+}

+

+static inline void perf_exit(void)

+{

+}

+#endif

+

+#endif

diff --git a/accel/tcg/perf.c b/accel/tcg/perf.c

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/accel/tcg/perf.c

@@ -XXX,XX +XXX,XX @@

+/*

+ * Linux perf perf-<pid>.map and jit-<pid>.dump integration.

+ *

+ * The jitdump spec can be found at [1].

+ *

+ * [1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/tools/perf/Documentation/jitdump-specification.txt

+ *

+ * SPDX-License-Identifier: GPL-2.0-or-later

+ */

+

+#include "qemu/osdep.h"

+#include "elf.h"

+#include "exec/exec-all.h"

+#include "qemu/timer.h"

+#include "tcg/tcg.h"

+

+#include "debuginfo.h"

+#include "perf.h"

+

+static FILE *safe_fopen_w(const char *path)

+{

+ int saved_errno;

+ FILE *f;

+ int fd;

+

+ /* Delete the old file, if any. */

+ unlink(path);

+

+ /* Avoid symlink attacks by using O_CREAT | O_EXCL. */

+ fd = open(path, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);

+ if (fd == -1) {

+ return NULL;

+ }

+

+ /* Convert fd to FILE*. */

+ f = fdopen(fd, "w");

+ if (f == NULL) {

+ saved_errno = errno;

+ close(fd);

+ errno = saved_errno;

+ return NULL;

+ }

+

+ return f;

+}

+

+static FILE *perfmap;

+

+void perf_enable_perfmap(void)

+{

+ char map_file[32];

+

+ snprintf(map_file, sizeof(map_file), "/tmp/perf-%d.map", getpid());

+ perfmap = safe_fopen_w(map_file);

+ if (perfmap == NULL) {

+ warn_report("Could not open %s: %s, proceeding without perfmap",

+ map_file, strerror(errno));

+ }

+}

+

+/* Get PC and size of code JITed for guest instruction #INSN. */

+static void get_host_pc_size(uintptr_t *host_pc, uint16_t *host_size,

+ const void *start, size_t insn)

+{

+ uint16_t start_off = insn ? tcg_ctx->gen_insn_end_off[insn - 1] : 0;

+

+ if (host_pc) {

+ *host_pc = (uintptr_t)start + start_off;

+ }

+ if (host_size) {

+ *host_size = tcg_ctx->gen_insn_end_off[insn] - start_off;

+ }

+}

+

+static const char *pretty_symbol(const struct debuginfo_query *q, size_t *len)

+{

+ static __thread char buf[64];

+ int tmp;

+

+ if (!q->symbol) {

+ tmp = snprintf(buf, sizeof(buf), "guest-0x%"PRIx64, q->address);

+ if (len) {

+ *len = MIN(tmp + 1, sizeof(buf));

+ }

+ return buf;

+ }

+

+ if (!q->offset) {

+ if (len) {

+ *len = strlen(q->symbol) + 1;

+ }

+ return q->symbol;

+ }

+

+ tmp = snprintf(buf, sizeof(buf), "%s+0x%"PRIx64, q->symbol, q->offset);

+ if (len) {

+ *len = MIN(tmp + 1, sizeof(buf));

+ }

+ return buf;

+}

+

+static void write_perfmap_entry(const void *start, size_t insn,

+ const struct debuginfo_query *q)

+{

+ uint16_t host_size;

+ uintptr_t host_pc;

+

+ get_host_pc_size(&host_pc, &host_size, start, insn);

+ fprintf(perfmap, "%"PRIxPTR" %"PRIx16" %s\n",

+ host_pc, host_size, pretty_symbol(q, NULL));

+}

+

+static FILE *jitdump;

+

+#define JITHEADER_MAGIC 0x4A695444

+#define JITHEADER_VERSION 1

+

+struct jitheader {

+ uint32_t magic;

+ uint32_t version;

+ uint32_t total_size;

+ uint32_t elf_mach;

+ uint32_t pad1;

+ uint32_t pid;

+ uint64_t timestamp;

+ uint64_t flags;

+};

+

+enum jit_record_type {

+ JIT_CODE_LOAD = 0,

+ JIT_CODE_DEBUG_INFO = 2,

+};

+

+struct jr_prefix {

+ uint32_t id;

+ uint32_t total_size;

+ uint64_t timestamp;

+};

+

+struct jr_code_load {

+ struct jr_prefix p;

+

+ uint32_t pid;

+ uint32_t tid;

+ uint64_t vma;

+ uint64_t code_addr;

+ uint64_t code_size;

+ uint64_t code_index;

+};

+

+struct debug_entry {

+ uint64_t addr;

+ int lineno;

+ int discrim;

+ const char name[];

+};

+

+struct jr_code_debug_info {

+ struct jr_prefix p;

+

+ uint64_t code_addr;

+ uint64_t nr_entry;

+ struct debug_entry entries[];

+};

+

+static uint32_t get_e_machine(void)

+{

+ Elf64_Ehdr elf_header;

+ FILE *exe;

+ size_t n;

+

+ QEMU_BUILD_BUG_ON(offsetof(Elf32_Ehdr, e_machine) !=

+ offsetof(Elf64_Ehdr, e_machine));

+

+ exe = fopen("/proc/self/exe", "r");

+ if (exe == NULL) {

+ return EM_NONE;

+ }

+

+ n = fread(&elf_header, sizeof(elf_header), 1, exe);

+ fclose(exe);

+ if (n != 1) {

+ return EM_NONE;

+ }

+

+ return elf_header.e_machine;

+}

+

+void perf_enable_jitdump(void)

+{

+ struct jitheader header;

+ char jitdump_file[32];

+ void *perf_marker;

+

+ if (!use_rt_clock) {

+ warn_report("CLOCK_MONOTONIC is not available, proceeding without jitdump");

+ return;