From nobody Fri May 3 16:13:15 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1538597226339149.75554762579907; Wed, 3 Oct 2018 13:07:06 -0700 (PDT) Received: from localhost ([::1]:52783 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1g7nQD-0003yh-0y for importer@patchew.org; Wed, 03 Oct 2018 16:07:01 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:41141) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1g7nOI-0002cU-Ob for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:05:03 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1g7nOF-0006wO-G0 for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:05:02 -0400 Received: from wout2-smtp.messagingengine.com ([64.147.123.25]:36467) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1g7nOF-0006u3-0g for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:04:59 -0400 Received: from compute4.internal (compute4.nyi.internal [10.202.2.44]) by mailout.west.internal (Postfix) with ESMTP id E8E57CEA; Wed, 3 Oct 2018 16:04:56 -0400 (EDT) Received: from mailfrontend2 ([10.202.2.163]) by compute4.internal (MEProxy); Wed, 03 Oct 2018 16:04:57 -0400 Received: from localhost (flamenco.cs.columbia.edu [128.59.20.216]) by mail.messagingengine.com (Postfix) with ESMTPA id 28A4D102E0; Wed, 3 Oct 2018 16:04:56 -0400 (EDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=braap.org; h= from:to:cc:subject:date:message-id:in-reply-to:references; s= mesmtp; bh=mkt190u36865Xg/4JVPuynMLAAAsS05mGUSCBk2HQAE=; b=NI6O4 sq2CZ3VeV+HcrBOh0IphUD1vcFLDht0FEzQsUbyDKCpXhYK4qmSPcVVE7W8GfWuq +HJLC+5vLaiNegY20Kvy0pOELJpQUBB1pn7LkF5FVEuIiV/TRm2e8r73RQuyADt9 +cUoUC33iNLxvunQOUMnRPlW9URnLFSZd6i1TE= DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d= messagingengine.com; h=cc:date:from:in-reply-to:message-id :references:subject:to:x-me-proxy:x-me-proxy:x-me-sender :x-me-sender:x-sasl-enc; s=fm3; bh=mkt190u36865Xg/4JVPuynMLAAAsS 05mGUSCBk2HQAE=; b=ZO+kmpIDkmYMURWOTJTL7yd2PUFcPa0Zzj4cGy0ixQUpb pP7JY0UTIHthBscSbH0F2HJ+c7ZN4lqrrUk8PllHOYXBAX6vHcghhRHz6vPgaeyc Bv4a7bQ07CcgaFu11L1b2o2t2qTLkNCFgGmHji7JRRxMSH41PM1yeXr1uXDLR5Xy T6fSdslDKK3ZV/fWcRUE7jJxLnH0zFq2EGG5VO7uIm90sRHPztW4YMjueIT+Q3sB yKz3AvYOVmGYQ9hpg3seF0dVjiej83ldD1bgiXnSA8d3/mm/9I1nWNDXOs4dRz91 Ln53a05Hk9b4Zy7kWqmPjTQ/3mRc5LSfOflHg2ftw== X-ME-Sender: X-ME-Proxy: From: "Emilio G. Cota" To: qemu-devel@nongnu.org Date: Wed, 3 Oct 2018 16:04:51 -0400 Message-Id: <20181003200454.18384-2-cota@braap.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20181003200454.18384-1-cota@braap.org> References: <20181003200454.18384-1-cota@braap.org> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 64.147.123.25 Subject: [Qemu-devel] [PATCH v2 1/4] exec: introduce tlb_init X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Paolo Bonzini , =?UTF-8?q?Alex=20Benn=C3=A9e?= , Richard Henderson Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Paves the way for the addition of a per-TLB lock. Signed-off-by: Emilio G. Cota Reviewed-by: Alex Benn=C3=A9e Tested-by: Alex Benn=C3=A9e --- include/exec/exec-all.h | 8 ++++++++ accel/tcg/cputlb.c | 4 ++++ exec.c | 1 + 3 files changed, 13 insertions(+) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 5f78125582..815e5b1e83 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -99,6 +99,11 @@ void cpu_address_space_init(CPUState *cpu, int asidx, =20 #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG) /* cputlb.c */ +/** + * tlb_init - initialize a CPU's TLB + * @cpu: CPU whose TLB should be initialized + */ +void tlb_init(CPUState *cpu); /** * tlb_flush_page: * @cpu: CPU whose TLB should be flushed @@ -258,6 +263,9 @@ void tlb_set_page(CPUState *cpu, target_ulong vaddr, void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_i= dx, uintptr_t retaddr); #else +static inline void tlb_init(CPUState *cpu) +{ +} static inline void tlb_flush_page(CPUState *cpu, target_ulong addr) { } diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index f4702ce91f..502eea2850 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -73,6 +73,10 @@ QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_c= pu_data)); QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16); #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1) =20 +void tlb_init(CPUState *cpu) +{ +} + /* flush_all_helper: run fn across all cpus * * If the wait flag is set then the src cpu's helper will be queued as diff --git a/exec.c b/exec.c index d0821e69aa..4fd831ef06 100644 --- a/exec.c +++ b/exec.c @@ -965,6 +965,7 @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp) tcg_target_initialized =3D true; cc->tcg_initialize(); } + tlb_init(cpu); =20 #ifndef CONFIG_USER_ONLY if (qdev_get_vmsd(DEVICE(cpu)) =3D=3D NULL) { --=20 2.17.1 From nobody Fri May 3 16:13:15 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1538597228195360.2991232386064; Wed, 3 Oct 2018 13:07:08 -0700 (PDT) Received: from localhost ([::1]:52784 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1g7nQI-00040L-VG for importer@patchew.org; Wed, 03 Oct 2018 16:07:07 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:41144) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1g7nOI-0002ca-Or for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:05:03 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1g7nOF-0006wD-AX for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:05:02 -0400 Received: from wout2-smtp.messagingengine.com ([64.147.123.25]:36181) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1g7nOE-0006u2-Tf for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:04:59 -0400 Received: from compute4.internal (compute4.nyi.internal [10.202.2.44]) by mailout.west.internal (Postfix) with ESMTP id 161C5D52; Wed, 3 Oct 2018 16:04:57 -0400 (EDT) Received: from mailfrontend2 ([10.202.2.163]) by compute4.internal (MEProxy); Wed, 03 Oct 2018 16:04:57 -0400 Received: from localhost (flamenco.cs.columbia.edu [128.59.20.216]) by mail.messagingengine.com (Postfix) with ESMTPA id 61941102E4; Wed, 3 Oct 2018 16:04:56 -0400 (EDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=braap.org; h= from:to:cc:subject:date:message-id:in-reply-to:references; s= mesmtp; bh=mOVOWodmMVYr697izTsN8/2ez+zLM5WHswazphP/Ftg=; b=sbyXH 0d5Sf2ymjEOig3Dq/DepSZRgc7aAojIEqw78YQkmywPYmyvql4ZkaXr6ctZoFRI7 9WpXUgCpvV7E8RvVG0Vo4nd/Yw0PDz4UYbAD5ci5sfibnDdyVFfz2Z5vdai8wb/i CWXhLqUUgbx/PBWa6uvlBxeyuAUciz+C08R8/M= DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d= messagingengine.com; h=cc:date:from:in-reply-to:message-id :references:subject:to:x-me-proxy:x-me-proxy:x-me-sender :x-me-sender:x-sasl-enc; s=fm3; bh=mOVOWodmMVYr697izTsN8/2ez+zLM 5WHswazphP/Ftg=; b=H4N8MiNWtoU0fXAefHQ7Bq3CzOyxIy5HWo3e02XS6mlnX SdeQGmNp2VbEbM6pdcYHFDaCgZM6LEt340ehPqaaO2l2H1nR7NtoTNkmZCl8VJ2P xkrHffW4zliRdVyl1QbUOtuvXlFApugUG4kq7klXMWdyZp0MOqqFNKQwt334Xe+V 5hNkm4I1y375IKBNbYuX98lfSLeUGQZAFiTd0Ml8abeQVNfnGF8Ju7WAJRh+Jrkp wsn0kw5qczUUt0D4lOBQt789lgU+Og4FWiY3oe+99tKTyZKVJxfjU/jUexe+3Cjc n6evDFZZBGvMn4yYlJyrZfLfhsGc8a2uuZI68qapw== X-ME-Sender: X-ME-Proxy: From: "Emilio G. Cota" To: qemu-devel@nongnu.org Date: Wed, 3 Oct 2018 16:04:52 -0400 Message-Id: <20181003200454.18384-3-cota@braap.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20181003200454.18384-1-cota@braap.org> References: <20181003200454.18384-1-cota@braap.org> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 64.147.123.25 Subject: [Qemu-devel] [PATCH v2 2/4] cputlb: fix assert_cpu_is_self macro X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Paolo Bonzini , =?UTF-8?q?Alex=20Benn=C3=A9e?= , Richard Henderson Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Signed-off-by: Emilio G. Cota Reviewed-by: Alex Benn=C3=A9e Reviewed-by: Richard Henderson Tested-by: Alex Benn=C3=A9e --- accel/tcg/cputlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 502eea2850..f6b388c961 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -58,9 +58,9 @@ } \ } while (0) =20 -#define assert_cpu_is_self(this_cpu) do { \ +#define assert_cpu_is_self(cpu) do { \ if (DEBUG_TLB_GATE) { \ - g_assert(!cpu->created || qemu_cpu_is_self(cpu)); \ + g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \ } \ } while (0) =20 --=20 2.17.1 From nobody Fri May 3 16:13:15 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1538597381727685.3417482043452; Wed, 3 Oct 2018 13:09:41 -0700 (PDT) Received: from localhost ([::1]:52797 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1g7nSm-0006rM-KX for importer@patchew.org; Wed, 03 Oct 2018 16:09:40 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:41143) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1g7nOI-0002cZ-Og for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:05:04 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1g7nOF-0006wu-Tm for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:05:02 -0400 Received: from wout2-smtp.messagingengine.com ([64.147.123.25]:39665) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1g7nOF-0006vK-De for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:04:59 -0400 Received: from compute4.internal (compute4.nyi.internal [10.202.2.44]) by mailout.west.internal (Postfix) with ESMTP id 2F45CC36; Wed, 3 Oct 2018 16:04:58 -0400 (EDT) Received: from mailfrontend2 ([10.202.2.163]) by compute4.internal (MEProxy); Wed, 03 Oct 2018 16:04:58 -0400 Received: from localhost (flamenco.cs.columbia.edu [128.59.20.216]) by mail.messagingengine.com (Postfix) with ESMTPA id 901DC102E7; Wed, 3 Oct 2018 16:04:56 -0400 (EDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=braap.org; h= from:to:cc:subject:date:message-id:in-reply-to:references; s= mesmtp; bh=pvxoibOCNbCSbPL2ayx1LHThTYqI/f1Xe8r4VTIOwn0=; b=Y4liK WY1fKH9E3DIpnScuqwC1K87NvEVye4c4tiOCBxbyMUq3dZQmG5KcvKNpvcA1MbGm FwZWhKUFjZ//0/+i6goMpu0yNuImEzwr6aRzhCxupfcFJAGkxIUKVuO9nUXowwxO pNDqzhWnqJVMAlYMDOTvTY0fmz7GrnmkdCvWEE= DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d= messagingengine.com; h=cc:date:from:in-reply-to:message-id :references:subject:to:x-me-proxy:x-me-proxy:x-me-sender :x-me-sender:x-sasl-enc; s=fm3; bh=pvxoibOCNbCSbPL2ayx1LHThTYqI/ f1Xe8r4VTIOwn0=; b=Niw+VaSXWrVRYhzOVA0F7BOM55MtPtR9WCBhBa2KuptJV zBpYu6CGPAV3FgmO+vJSEHiMM9CEn4cd4THZRLCZcu1tnQTmMlmxIBb4gAi1Vk2H VyDcEpqQ+ybySflCy1Vj61Gmh3is3DEIIVZ8SXJHRxsyWilvv1gPOY1W/kZDyBZA O9Pvm6a4VpD6OmTYqMYyjmmA9TUcSpRhp5Q/3ry0+0fZbvitdE64lH25fGjL63AH Y+SBZGSCAHRNkXtwiRp8WCvlTsVKfApnwklgjCF+nUp2fV9GVqLKaCRpV4/TFHI8 3fV53MvGi+oPiaJyP44VsN8OIvzZvOTs1RMPBkwsA== X-ME-Sender: X-ME-Proxy: From: "Emilio G. Cota" To: qemu-devel@nongnu.org Date: Wed, 3 Oct 2018 16:04:53 -0400 Message-Id: <20181003200454.18384-4-cota@braap.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20181003200454.18384-1-cota@braap.org> References: <20181003200454.18384-1-cota@braap.org> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 64.147.123.25 Subject: [Qemu-devel] [PATCH v2 3/4] cputlb: serialize tlb updates with env->tlb_lock X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Paolo Bonzini , =?UTF-8?q?Alex=20Benn=C3=A9e?= , Richard Henderson Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Currently we rely on atomic operations for cross-CPU invalidations. There are two cases that these atomics miss: cross-CPU invalidations can race with either (1) vCPU threads flushing their TLB, which happens via memset, or (2) vCPUs calling tlb_reset_dirty on their TLB, which updates .addr_write with a regular store. This results in undefined behaviour, since we're mixing regular and atomic ops on concurrent accesses. Fix it by using tlb_lock, a per-vCPU lock. All updaters of tlb_table and the corresponding victim cache now hold the lock. The readers that do not hold tlb_lock must use atomic reads when reading .addr_write, since this field can be updated by other threads; the conversion to atomic reads is done in the next patch. Note that an alternative fix would be to expand the use of atomic ops. However, in the case of TLB flushes this would have a huge performance impact, since (1) TLB flushes can happen very frequently and (2) we currently use a full memory barrier to flush each TLB entry, and a TLB has many entries. Instead, acquiring the lock is barely slower than a full memory barrier since it is uncontended, and with a single lock acquisition we can flush the entire TLB. Signed-off-by: Emilio G. Cota Tested-by: Alex Benn=C3=A9e --- include/exec/cpu-defs.h | 2 + accel/tcg/cputlb.c | 153 ++++++++++++++++++++++------------------ 2 files changed, 87 insertions(+), 68 deletions(-) diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index a171ffc1a4..bcc40c8ef5 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -142,6 +142,8 @@ typedef struct CPUIOTLBEntry { =20 #define CPU_COMMON_TLB \ /* The meaning of the MMU modes is defined in the target code. */ \ + /* tlb_lock serializes updates to tlb_table and tlb_v_table */ \ + QemuMutex tlb_lock; \ CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE]; \ CPUTLBEntry tlb_v_table[NB_MMU_MODES][CPU_VTLB_SIZE]; \ CPUIOTLBEntry iotlb[NB_MMU_MODES][CPU_TLB_SIZE]; \ diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index f6b388c961..142a9cdf9e 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -75,6 +75,9 @@ QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16); =20 void tlb_init(CPUState *cpu) { + CPUArchState *env =3D cpu->env_ptr; + + qemu_mutex_init(&env->tlb_lock); } =20 /* flush_all_helper: run fn across all cpus @@ -129,8 +132,17 @@ static void tlb_flush_nocheck(CPUState *cpu) atomic_set(&env->tlb_flush_count, env->tlb_flush_count + 1); tlb_debug("(count: %zu)\n", tlb_flush_count()); =20 + /* + * tlb_table/tlb_v_table updates from any thread must hold tlb_lock. + * However, updates from the owner thread (as is the case here; see the + * above assert_cpu_is_self) do not need atomic_set because all reads + * that do not hold the lock are performed by the same owner thread. + */ + qemu_mutex_lock(&env->tlb_lock); memset(env->tlb_table, -1, sizeof(env->tlb_table)); memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table)); + qemu_mutex_unlock(&env->tlb_lock); + cpu_tb_jmp_cache_clear(cpu); =20 env->vtlb_index =3D 0; @@ -182,6 +194,7 @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cp= u, run_on_cpu_data data) =20 tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask); =20 + qemu_mutex_lock(&env->tlb_lock); for (mmu_idx =3D 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { =20 if (test_bit(mmu_idx, &mmu_idx_bitmask)) { @@ -191,6 +204,7 @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cp= u, run_on_cpu_data data) memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[= 0])); } } + qemu_mutex_unlock(&env->tlb_lock); =20 cpu_tb_jmp_cache_clear(cpu); =20 @@ -247,22 +261,36 @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *= tlb_entry, tlb_hit_page(tlb_entry->addr_code, page); } =20 -static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong pa= ge) +/* Called with tlb_lock held */ +static inline void tlb_flush_entry_locked(CPUTLBEntry *tlb_entry, + target_ulong page) { if (tlb_hit_page_anyprot(tlb_entry, page)) { memset(tlb_entry, -1, sizeof(*tlb_entry)); } } =20 -static inline void tlb_flush_vtlb_page(CPUArchState *env, int mmu_idx, - target_ulong page) +/* Called with tlb_lock held */ +static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_i= dx, + target_ulong page) { int k; + + assert_cpu_is_self(ENV_GET_CPU(env)); for (k =3D 0; k < CPU_VTLB_SIZE; k++) { - tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], page); + tlb_flush_entry_locked(&env->tlb_v_table[mmu_idx][k], page); } } =20 +static inline void tlb_flush_vtlb_page(CPUArchState *env, int mmu_idx, + target_ulong page) +{ + assert_cpu_is_self(ENV_GET_CPU(env)); + qemu_mutex_lock(&env->tlb_lock); + tlb_flush_vtlb_page_locked(env, mmu_idx, page); + qemu_mutex_unlock(&env->tlb_lock); +} + static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data) { CPUArchState *env =3D cpu->env_ptr; @@ -286,10 +314,12 @@ static void tlb_flush_page_async_work(CPUState *cpu, = run_on_cpu_data data) =20 addr &=3D TARGET_PAGE_MASK; i =3D (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); + qemu_mutex_lock(&env->tlb_lock); for (mmu_idx =3D 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr); - tlb_flush_vtlb_page(env, mmu_idx, addr); + tlb_flush_entry_locked(&env->tlb_table[mmu_idx][i], addr); + tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } + qemu_mutex_unlock(&env->tlb_lock); =20 tb_flush_jmp_cache(cpu, addr); } @@ -326,12 +356,14 @@ static void tlb_flush_page_by_mmuidx_async_work(CPUSt= ate *cpu, tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n", page, addr, mmu_idx_bitmap); =20 + qemu_mutex_lock(&env->tlb_lock); for (mmu_idx =3D 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { if (test_bit(mmu_idx, &mmu_idx_bitmap)) { - tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr); - tlb_flush_vtlb_page(env, mmu_idx, addr); + tlb_flush_entry_locked(&env->tlb_table[mmu_idx][page], addr); + tlb_flush_vtlb_page_locked(env, mmu_idx, addr); } } + qemu_mutex_unlock(&env->tlb_lock); =20 tb_flush_jmp_cache(cpu, addr); } @@ -454,72 +486,49 @@ void tlb_unprotect_code(ram_addr_t ram_addr) * most usual is detecting writes to code regions which may invalidate * generated code. * - * Because we want other vCPUs to respond to changes straight away we - * update the te->addr_write field atomically. If the TLB entry has - * been changed by the vCPU in the mean time we skip the update. + * Other vCPUs might be reading their TLBs during guest execution, so we u= pdate + * te->addr_write with atomic_set. We don't need to worry about this for + * oversized guests as MTTCG is disabled for them. * - * As this function uses atomic accesses we also need to ensure - * updates to tlb_entries follow the same access rules. We don't need - * to worry about this for oversized guests as MTTCG is disabled for - * them. + * Called with tlb_lock held. */ - -static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start, - uintptr_t length) +static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry, + uintptr_t start, uintptr_t length) { -#if TCG_OVERSIZED_GUEST uintptr_t addr =3D tlb_entry->addr_write; =20 if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) =3D=3D 0) { addr &=3D TARGET_PAGE_MASK; addr +=3D tlb_entry->addend; if ((addr - start) < length) { +#if TCG_OVERSIZED_GUEST tlb_entry->addr_write |=3D TLB_NOTDIRTY; - } - } #else - /* paired with atomic_mb_set in tlb_set_page_with_attrs */ - uintptr_t orig_addr =3D atomic_mb_read(&tlb_entry->addr_write); - uintptr_t addr =3D orig_addr; - - if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) =3D=3D 0) { - addr &=3D TARGET_PAGE_MASK; - addr +=3D atomic_read(&tlb_entry->addend); - if ((addr - start) < length) { - uintptr_t notdirty_addr =3D orig_addr | TLB_NOTDIRTY; - atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_add= r); + atomic_set(&tlb_entry->addr_write, + tlb_entry->addr_write | TLB_NOTDIRTY); +#endif } } -#endif } =20 -/* For atomic correctness when running MTTCG we need to use the right - * primitives when copying entries */ -static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s, - bool atomic_set) +/* Called with tlb_lock held */ +static void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s) { -#if TCG_OVERSIZED_GUEST *d =3D *s; -#else - if (atomic_set) { - d->addr_read =3D s->addr_read; - d->addr_code =3D s->addr_code; - atomic_set(&d->addend, atomic_read(&s->addend)); - /* Pairs with flag setting in tlb_reset_dirty_range */ - atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write)); - } else { - d->addr_read =3D s->addr_read; - d->addr_write =3D atomic_read(&s->addr_write); - d->addr_code =3D s->addr_code; - d->addend =3D atomic_read(&s->addend); - } -#endif +} + +static void copy_tlb_helper(CPUArchState *env, CPUTLBEntry *d, CPUTLBEntry= *s) +{ + assert_cpu_is_self(ENV_GET_CPU(env)); + qemu_mutex_lock(&env->tlb_lock); + copy_tlb_helper_locked(d, s); + qemu_mutex_unlock(&env->tlb_lock); } =20 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of - * the target vCPU). As such care needs to be taken that we don't - * dangerously race with another vCPU update. The only thing actually - * updated is the target TLB entry ->addr_write flags. + * the target vCPU). + * We must take tlb_lock to avoid racing with another vCPU update. The only + * thing actually updated is the target TLB entry ->addr_write flags. */ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length) { @@ -528,22 +537,26 @@ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1= , ram_addr_t length) int mmu_idx; =20 env =3D cpu->env_ptr; + qemu_mutex_lock(&env->tlb_lock); for (mmu_idx =3D 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { unsigned int i; =20 for (i =3D 0; i < CPU_TLB_SIZE; i++) { - tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i], - start1, length); + tlb_reset_dirty_range_locked(&env->tlb_table[mmu_idx][i], star= t1, + length); } =20 for (i =3D 0; i < CPU_VTLB_SIZE; i++) { - tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i], - start1, length); + tlb_reset_dirty_range_locked(&env->tlb_v_table[mmu_idx][i], st= art1, + length); } } + qemu_mutex_unlock(&env->tlb_lock); } =20 -static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vad= dr) +/* Called with tlb_lock held */ +static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry, + target_ulong vaddr) { if (tlb_entry->addr_write =3D=3D (vaddr | TLB_NOTDIRTY)) { tlb_entry->addr_write =3D vaddr; @@ -562,16 +575,18 @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr) =20 vaddr &=3D TARGET_PAGE_MASK; i =3D (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); + qemu_mutex_lock(&env->tlb_lock); for (mmu_idx =3D 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr); + tlb_set_dirty1_locked(&env->tlb_table[mmu_idx][i], vaddr); } =20 for (mmu_idx =3D 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { int k; for (k =3D 0; k < CPU_VTLB_SIZE; k++) { - tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr); + tlb_set_dirty1_locked(&env->tlb_v_table[mmu_idx][k], vaddr); } } + qemu_mutex_unlock(&env->tlb_lock); } =20 /* Our TLB does not support large pages, so remember the area covered by @@ -677,7 +692,7 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulon= g vaddr, CPUTLBEntry *tv =3D &env->tlb_v_table[mmu_idx][vidx]; =20 /* Evict the old entry into the victim tlb. */ - copy_tlb_helper(tv, te, true); + copy_tlb_helper(env, tv, te); env->iotlb_v[mmu_idx][vidx] =3D env->iotlb[mmu_idx][index]; } =20 @@ -729,9 +744,7 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulon= g vaddr, } } =20 - /* Pairs with flag setting in tlb_reset_dirty_range */ - copy_tlb_helper(te, &tn, true); - /* atomic_mb_set(&te->addr_write, write_address); */ + copy_tlb_helper(env, te, &tn); } =20 /* Add a new TLB entry, but without specifying the memory @@ -895,6 +908,8 @@ static bool victim_tlb_hit(CPUArchState *env, size_t mm= u_idx, size_t index, size_t elt_ofs, target_ulong page) { size_t vidx; + + assert_cpu_is_self(ENV_GET_CPU(env)); for (vidx =3D 0; vidx < CPU_VTLB_SIZE; ++vidx) { CPUTLBEntry *vtlb =3D &env->tlb_v_table[mmu_idx][vidx]; target_ulong cmp =3D *(target_ulong *)((uintptr_t)vtlb + elt_ofs); @@ -903,9 +918,11 @@ static bool victim_tlb_hit(CPUArchState *env, size_t m= mu_idx, size_t index, /* Found entry in victim tlb, swap tlb and iotlb. */ CPUTLBEntry tmptlb, *tlb =3D &env->tlb_table[mmu_idx][index]; =20 - copy_tlb_helper(&tmptlb, tlb, false); - copy_tlb_helper(tlb, vtlb, true); - copy_tlb_helper(vtlb, &tmptlb, true); + qemu_mutex_lock(&env->tlb_lock); + copy_tlb_helper_locked(&tmptlb, tlb); + copy_tlb_helper_locked(tlb, vtlb); + copy_tlb_helper_locked(vtlb, &tmptlb); + qemu_mutex_unlock(&env->tlb_lock); =20 CPUIOTLBEntry tmpio, *io =3D &env->iotlb[mmu_idx][index]; CPUIOTLBEntry *vio =3D &env->iotlb_v[mmu_idx][vidx]; --=20 2.17.1 From nobody Fri May 3 16:13:15 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1538597249848357.3467976119638; Wed, 3 Oct 2018 13:07:29 -0700 (PDT) Received: from localhost ([::1]:52785 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1g7nQb-0004HA-OY for importer@patchew.org; Wed, 03 Oct 2018 16:07:28 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:41142) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1g7nOI-0002cV-Od for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:05:04 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1g7nOG-0006xD-1C for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:05:02 -0400 Received: from wout2-smtp.messagingengine.com ([64.147.123.25]:50577) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1g7nOF-0006vI-Hz for qemu-devel@nongnu.org; Wed, 03 Oct 2018 16:04:59 -0400 Received: from compute4.internal (compute4.nyi.internal [10.202.2.44]) by mailout.west.internal (Postfix) with ESMTP id 24C7AC32; Wed, 3 Oct 2018 16:04:58 -0400 (EDT) Received: from mailfrontend2 ([10.202.2.163]) by compute4.internal (MEProxy); Wed, 03 Oct 2018 16:04:58 -0400 Received: from localhost (flamenco.cs.columbia.edu [128.59.20.216]) by mail.messagingengine.com (Postfix) with ESMTPA id C06A9102E8; Wed, 3 Oct 2018 16:04:56 -0400 (EDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=braap.org; h= from:to:cc:subject:date:message-id:in-reply-to:references; s= mesmtp; bh=/YlhVzvrDuauNmHQ6278YCjqdnRnpHkkXeT18uNWN+w=; b=bq3CZ ytJl6A2U6Dhe1mrbtqkY4c+7Z2zgx+4F1FzybSqwlsRYvK0brh+Pm8ESZm0S8ipl X/Qs28USVMcwZjg4VcWwYAHFdnUySz4LFLKueeX/R+p3kEybboTT+9STs0bTgQzl 83vf6zMygVbOv2WtisxUhLrouHzf0yKUBmpq4E= DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d= messagingengine.com; h=cc:date:from:in-reply-to:message-id :references:subject:to:x-me-proxy:x-me-proxy:x-me-sender :x-me-sender:x-sasl-enc; s=fm3; bh=/YlhVzvrDuauNmHQ6278YCjqdnRnp HkkXeT18uNWN+w=; b=SNbbu1L3Uxeyqfni4Nyu+h1r010eE5U+oHonJYGMzk9gL J/p9EJGVO3RpkNL1MF7LTMZVyVwavgWF+am1r0AjhEyoEwzStoposQXR4ubH+hE/ JJdl35MH44OtM3vNbIhc8Nbx1fs/a5oO9f8yzF6d7TRf+DbwZeHDHXha1/0xHPGW Ci1LzJF60+SQLT8iZqPFTCDMfXHQZy3D+8n9d6ffGchTv2iKqi4PVsRULOXCChkT lyEDuTRX8WpHc9LC0QGtioyHrTxiF/aowh6rV3DnTdEOjGvFwy3bo3hcJDXQ2pQu Tm8OCJmzfKp80bIVR1E2OFCKxoMh1X7kRjpqVasOg== X-ME-Sender: X-ME-Proxy: From: "Emilio G. Cota" To: qemu-devel@nongnu.org Date: Wed, 3 Oct 2018 16:04:54 -0400 Message-Id: <20181003200454.18384-5-cota@braap.org> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20181003200454.18384-1-cota@braap.org> References: <20181003200454.18384-1-cota@braap.org> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 64.147.123.25 Subject: [Qemu-devel] [PATCH v2 4/4] cputlb: read CPUTLBEntry.addr_write atomically X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Paolo Bonzini , =?UTF-8?q?Alex=20Benn=C3=A9e?= , Richard Henderson Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Updates can come from other threads, so readers that do not take tlb_lock must use atomic_read to avoid undefined behaviour (UB). This and the previous commit result in a small performance decrease, but this is a fair price for removing UB. Host: Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz - Before: Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 run= s): 7482.981146 task-clock (msec) # 0.998 CPUs utilized = ( +- 0.09% ) 31,565,219,958 cycles # 4.218 GHz = ( +- 0.09% ) 57,102,517,194 instructions # 1.81 insns per cycl= e ( +- 0.07% ) 10,255,768,012 branches # 1370.546 M/sec = ( +- 0.07% ) 172,980,542 branch-misses # 1.69% of all branche= s ( +- 0.11% ) 7.494710830 seconds time elapsed = ( +- 0.09% ) - After: Performance counter stats for 'taskset -c 0 ../img/aarch64/die.sh' (10 run= s): 7649.735155 task-clock (msec) # 0.999 CPUs utilized = ( +- 0.13% ) 32,262,593,483 cycles # 4.217 GHz = ( +- 0.13% ) 58,487,065,236 instructions # 1.81 insns per cycl= e ( +- 0.06% ) 10,561,549,557 branches # 1380.643 M/sec = ( +- 0.06% ) 173,995,793 branch-misses # 1.65% of all branche= s ( +- 0.12% ) 7.660611466 seconds time elapsed = ( +- 0.13% ) That is, a ~2% slowdown for the aarch64 bootup+shutdown test. Signed-off-by: Emilio G. Cota Tested-by: Alex Benn=C3=A9e --- accel/tcg/softmmu_template.h | 16 ++++++++++------ include/exec/cpu_ldst.h | 2 +- include/exec/cpu_ldst_template.h | 2 +- accel/tcg/cputlb.c | 15 +++++++++------ 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/accel/tcg/softmmu_template.h b/accel/tcg/softmmu_template.h index f060a693d4..1e50263871 100644 --- a/accel/tcg/softmmu_template.h +++ b/accel/tcg/softmmu_template.h @@ -277,7 +277,8 @@ void helper_le_st_name(CPUArchState *env, target_ulong = addr, DATA_TYPE val, { unsigned mmu_idx =3D get_mmuidx(oi); int index =3D (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr =3D env->tlb_table[mmu_idx][index].addr_write; + target_ulong tlb_addr =3D + atomic_read(&env->tlb_table[mmu_idx][index].addr_write); unsigned a_bits =3D get_alignment_bits(get_memop(oi)); uintptr_t haddr; =20 @@ -292,7 +293,8 @@ void helper_le_st_name(CPUArchState *env, target_ulong = addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr =3D env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVA= LID_MASK; + tlb_addr =3D atomic_read(&env->tlb_table[mmu_idx][index].addr_writ= e) & + ~TLB_INVALID_MASK; } =20 /* Handle an IO access. */ @@ -321,7 +323,7 @@ void helper_le_st_name(CPUArchState *env, target_ulong = addr, DATA_TYPE val, cannot evict the first. */ page2 =3D (addr + DATA_SIZE) & TARGET_PAGE_MASK; index2 =3D (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr2 =3D env->tlb_table[mmu_idx][index2].addr_write; + tlb_addr2 =3D atomic_read(&env->tlb_table[mmu_idx][index2].addr_wr= ite); if (!tlb_hit_page(tlb_addr2, page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, @@ -354,7 +356,8 @@ void helper_be_st_name(CPUArchState *env, target_ulong = addr, DATA_TYPE val, { unsigned mmu_idx =3D get_mmuidx(oi); int index =3D (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr =3D env->tlb_table[mmu_idx][index].addr_write; + target_ulong tlb_addr =3D + atomic_read(&env->tlb_table[mmu_idx][index].addr_write); unsigned a_bits =3D get_alignment_bits(get_memop(oi)); uintptr_t haddr; =20 @@ -369,7 +372,8 @@ void helper_be_st_name(CPUArchState *env, target_ulong = addr, DATA_TYPE val, tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr =3D env->tlb_table[mmu_idx][index].addr_write & ~TLB_INVA= LID_MASK; + tlb_addr =3D atomic_read(&env->tlb_table[mmu_idx][index].addr_writ= e) & + ~TLB_INVALID_MASK; } =20 /* Handle an IO access. */ @@ -398,7 +402,7 @@ void helper_be_st_name(CPUArchState *env, target_ulong = addr, DATA_TYPE val, cannot evict the first. */ page2 =3D (addr + DATA_SIZE) & TARGET_PAGE_MASK; index2 =3D (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr2 =3D env->tlb_table[mmu_idx][index2].addr_write; + tlb_addr2 =3D atomic_read(&env->tlb_table[mmu_idx][index2].addr_wr= ite); if (!tlb_hit_page(tlb_addr2, page2) && !VICTIM_TLB_HIT(addr_write, page2)) { tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h index 41ed0526e2..9581587ce1 100644 --- a/include/exec/cpu_ldst.h +++ b/include/exec/cpu_ldst.h @@ -426,7 +426,7 @@ static inline void *tlb_vaddr_to_host(CPUArchState *env= , abi_ptr addr, tlb_addr =3D tlbentry->addr_read; break; case 1: - tlb_addr =3D tlbentry->addr_write; + tlb_addr =3D atomic_read(&tlbentry->addr_write); break; case 2: tlb_addr =3D tlbentry->addr_code; diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_templ= ate.h index 4db2302962..ba7a11123c 100644 --- a/include/exec/cpu_ldst_template.h +++ b/include/exec/cpu_ldst_template.h @@ -176,7 +176,7 @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArc= hState *env, addr =3D ptr; page_index =3D (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); mmu_idx =3D CPU_MMU_INDEX; - if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write !=3D + if (unlikely(atomic_read(&env->tlb_table[mmu_idx][page_index].addr_wri= te) !=3D (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { oi =3D make_memop_idx(SHIFT, mmu_idx); glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi, diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 142a9cdf9e..adbeda0d3b 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -257,7 +257,7 @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tl= b_entry, target_ulong page) { return tlb_hit_page(tlb_entry->addr_read, page) || - tlb_hit_page(tlb_entry->addr_write, page) || + tlb_hit_page(atomic_read(&tlb_entry->addr_write), page) || tlb_hit_page(tlb_entry->addr_code, page); } =20 @@ -863,7 +863,7 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry = *iotlbentry, tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr); =20 index =3D (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - tlb_addr =3D env->tlb_table[mmu_idx][index].addr_write; + tlb_addr =3D atomic_read(&env->tlb_table[mmu_idx][index].addr_writ= e); if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) { /* RAM access */ uintptr_t haddr =3D addr + env->tlb_table[mmu_idx][index].adde= nd; @@ -912,7 +912,9 @@ static bool victim_tlb_hit(CPUArchState *env, size_t mm= u_idx, size_t index, assert_cpu_is_self(ENV_GET_CPU(env)); for (vidx =3D 0; vidx < CPU_VTLB_SIZE; ++vidx) { CPUTLBEntry *vtlb =3D &env->tlb_v_table[mmu_idx][vidx]; - target_ulong cmp =3D *(target_ulong *)((uintptr_t)vtlb + elt_ofs); + /* elt_ofs might correspond to .addr_write, so use atomic_read */ + target_ulong cmp =3D + atomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs)); =20 if (cmp =3D=3D page) { /* Found entry in victim tlb, swap tlb and iotlb. */ @@ -984,7 +986,8 @@ void probe_write(CPUArchState *env, target_ulong addr, = int size, int mmu_idx, uintptr_t retaddr) { int index =3D (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - target_ulong tlb_addr =3D env->tlb_table[mmu_idx][index].addr_write; + target_ulong tlb_addr =3D + atomic_read(&env->tlb_table[mmu_idx][index].addr_write); =20 if (!tlb_hit(tlb_addr, addr)) { /* TLB entry is for a different page */ @@ -1004,7 +1007,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, tar= get_ulong addr, size_t mmu_idx =3D get_mmuidx(oi); size_t index =3D (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); CPUTLBEntry *tlbe =3D &env->tlb_table[mmu_idx][index]; - target_ulong tlb_addr =3D tlbe->addr_write; + target_ulong tlb_addr =3D atomic_read(&tlbe->addr_write); TCGMemOp mop =3D get_memop(oi); int a_bits =3D get_alignment_bits(mop); int s_bits =3D mop & MO_SIZE; @@ -1035,7 +1038,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, tar= get_ulong addr, tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_STORE, mmu_idx, retaddr); } - tlb_addr =3D tlbe->addr_write & ~TLB_INVALID_MASK; + tlb_addr =3D atomic_read(&tlbe->addr_write) & ~TLB_INVALID_MASK; } =20 /* Notice an IO access or a needs-MMU-lookup access */ --=20 2.17.1