[Qemu-devel] [PATCH RFC] s390x/kvm: call cpu_synchronize_state() on every kvm_arch_handle_exit()

David Hildenbrand posted 1 patch 6 years ago
Patches applied successfully (tree, apply log)
git fetch https://github.com/patchew-project/qemu tags/patchew/20180326092036.12780-1-david@redhat.com
Test checkpatch passed
Test docker-build@min-glib passed
Test docker-mingw@fedora passed
Test docker-quick@centos6 passed
Test s390x passed
hw/s390x/s390-pci-inst.c |  8 --------
target/s390x/kvm.c       | 20 ++------------------
2 files changed, 2 insertions(+), 26 deletions(-)
[Qemu-devel] [PATCH RFC] s390x/kvm: call cpu_synchronize_state() on every kvm_arch_handle_exit()
Posted by David Hildenbrand 6 years ago
Manually having to use cpu_synchronize_state() is error prone. I don't
think that the performance impact is that huge if we simply synchronize
the state on every kvm_arch_handle_exit() call. This makes the code
easier to maintain.

We now also call it (although not neded) for
- KVM_EXIT_S390_RESET -> s390_reipl_request()
- KVM_EXIT_DEBUG -> kvm_arch_handle_debug_exit()
- unmanagable/unimplemented intercepts
- ICPT_WAITPSW -> s390_handle_wait() -> cpu gets halted
- ICPT_CPU_STOP -> do_stop_interrupt() -> cpu gets halted
- Scenarios where we inject an operation exception
- handle_sigp() on the source CPU
- handle_stsi()

I don't think any of these are performance critical. Especially as we
have all information directly contained in kvm_run, there are no
additional IOCTLs to issue on modern kernels.

Remaining places (for s390x) are in
- target/s390x/sigp.c, on the target CPU
- target/s390x/cpu.c:s390_cpu_get_crash_info()

Signed-off-by: David Hildenbrand <david@redhat.com>
---
 hw/s390x/s390-pci-inst.c |  8 --------
 target/s390x/kvm.c       | 20 ++------------------
 2 files changed, 2 insertions(+), 26 deletions(-)

diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 3fcc330fe3..02a815fd31 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -155,8 +155,6 @@ int clp_service_call(S390CPU *cpu, uint8_t r2, uintptr_t ra)
     S390pciState *s = s390_get_phb();
     int i;
 
-    cpu_synchronize_state(CPU(cpu));
-
     if (env->psw.mask & PSW_MASK_PSTATE) {
         s390_program_interrupt(env, PGM_PRIVILEGED, 4, ra);
         return 0;
@@ -389,8 +387,6 @@ int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
     uint32_t fh;
     uint8_t pcias;
 
-    cpu_synchronize_state(CPU(cpu));
-
     if (env->psw.mask & PSW_MASK_PSTATE) {
         s390_program_interrupt(env, PGM_PRIVILEGED, 4, ra);
         return 0;
@@ -487,8 +483,6 @@ int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
     uint32_t fh;
     uint8_t pcias;
 
-    cpu_synchronize_state(CPU(cpu));
-
     if (env->psw.mask & PSW_MASK_PSTATE) {
         s390_program_interrupt(env, PGM_PRIVILEGED, 4, ra);
         return 0;
@@ -620,8 +614,6 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2, uintptr_t ra)
     S390IOTLBEntry entry;
     hwaddr start, end;
 
-    cpu_synchronize_state(CPU(cpu));
-
     if (env->psw.mask & PSW_MASK_PSTATE) {
         s390_program_interrupt(env, PGM_PRIVILEGED, 4, ra);
         return 0;
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index f570896dc1..7de861ab34 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -1081,7 +1081,6 @@ static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
     uint32_t code;
     int r = 0;
 
-    cpu_synchronize_state(CPU(cpu));
     sccb = env->regs[ipbh0 & 0xf];
     code = env->regs[(ipbh0 & 0xf0) >> 4];
 
@@ -1101,8 +1100,6 @@ static int handle_b2(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
     int rc = 0;
     uint16_t ipbh0 = (run->s390_sieic.ipb & 0xffff0000) >> 16;
 
-    cpu_synchronize_state(CPU(cpu));
-
     switch (ipa1) {
     case PRIV_B2_XSCH:
         ioinst_handle_xsch(cpu, env->regs[1], RA_IGNORED);
@@ -1248,7 +1245,6 @@ static int kvm_stpcifc_service_call(S390CPU *cpu, struct kvm_run *run)
     uint8_t ar;
 
     if (s390_has_feat(S390_FEAT_ZPCI)) {
-        cpu_synchronize_state(CPU(cpu));
         fiba = get_base_disp_rxy(cpu, run, &ar);
 
         return stpcifc_service_call(cpu, r1, fiba, ar, RA_IGNORED);
@@ -1266,7 +1262,6 @@ static int kvm_sic_service_call(S390CPU *cpu, struct kvm_run *run)
     uint16_t mode;
     int r;
 
-    cpu_synchronize_state(CPU(cpu));
     mode = env->regs[r1] & 0xffff;
     isc = (env->regs[r3] >> 27) & 0x7;
     r = css_do_sic(env, isc, mode);
@@ -1297,7 +1292,6 @@ static int kvm_pcistb_service_call(S390CPU *cpu, struct kvm_run *run)
     uint8_t ar;
 
     if (s390_has_feat(S390_FEAT_ZPCI)) {
-        cpu_synchronize_state(CPU(cpu));
         gaddr = get_base_disp_rsy(cpu, run, &ar);
 
         return pcistb_service_call(cpu, r1, r3, gaddr, ar, RA_IGNORED);
@@ -1313,7 +1307,6 @@ static int kvm_mpcifc_service_call(S390CPU *cpu, struct kvm_run *run)
     uint8_t ar;
 
     if (s390_has_feat(S390_FEAT_ZPCI)) {
-        cpu_synchronize_state(CPU(cpu));
         fiba = get_base_disp_rxy(cpu, run, &ar);
 
         return mpcifc_service_call(cpu, r1, fiba, ar, RA_IGNORED);
@@ -1401,7 +1394,6 @@ static int handle_hypercall(S390CPU *cpu, struct kvm_run *run)
     CPUS390XState *env = &cpu->env;
     int ret;
 
-    cpu_synchronize_state(CPU(cpu));
     ret = s390_virtio_hypercall(env);
     if (ret == -EINVAL) {
         kvm_s390_program_interrupt(cpu, PGM_SPECIFICATION);
@@ -1416,7 +1408,6 @@ static void kvm_handle_diag_288(S390CPU *cpu, struct kvm_run *run)
     uint64_t r1, r3;
     int rc;
 
-    cpu_synchronize_state(CPU(cpu));
     r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
     r3 = run->s390_sieic.ipa & 0x000f;
     rc = handle_diag_288(&cpu->env, r1, r3);
@@ -1429,7 +1420,6 @@ static void kvm_handle_diag_308(S390CPU *cpu, struct kvm_run *run)
 {
     uint64_t r1, r3;
 
-    cpu_synchronize_state(CPU(cpu));
     r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
     r3 = run->s390_sieic.ipa & 0x000f;
     handle_diag_308(&cpu->env, r1, r3, RA_IGNORED);
@@ -1440,8 +1430,6 @@ static int handle_sw_breakpoint(S390CPU *cpu, struct kvm_run *run)
     CPUS390XState *env = &cpu->env;
     unsigned long pc;
 
-    cpu_synchronize_state(CPU(cpu));
-
     pc = env->psw.addr - sw_bp_ilen;
     if (kvm_find_sw_breakpoint(CPU(cpu), pc)) {
         env->psw.addr = pc;
@@ -1493,8 +1481,6 @@ static int kvm_s390_handle_sigp(S390CPU *cpu, uint8_t ipa1, uint32_t ipb)
     int ret;
     uint8_t order;
 
-    cpu_synchronize_state(CPU(cpu));
-
     /* get order code */
     order = decode_basedisp_rs(env, ipb, NULL) & SIGP_ORDER_MASK;
 
@@ -1556,7 +1542,6 @@ static int handle_oper_loop(S390CPU *cpu, struct kvm_run *run)
     CPUState *cs = CPU(cpu);
     PSW oldpsw, newpsw;
 
-    cpu_synchronize_state(cs);
     newpsw.mask = ldq_phys(cs->as, cpu->env.psa +
                            offsetof(LowCore, program_new_psw));
     newpsw.addr = ldq_phys(cs->as, cpu->env.psa +
@@ -1609,7 +1594,6 @@ static int handle_intercept(S390CPU *cpu)
             break;
         case ICPT_WAITPSW:
             /* disabled wait, since enabled wait is handled in kernel */
-            cpu_synchronize_state(cs);
             s390_handle_wait(cpu);
             r = EXCP_HALTED;
             break;
@@ -1651,8 +1635,6 @@ static int handle_tsch(S390CPU *cpu)
     struct kvm_run *run = cs->kvm_run;
     int ret;
 
-    cpu_synchronize_state(cs);
-
     ret = ioinst_handle_tsch(cpu, cpu->env.regs[1], run->s390_tsch.ipb,
                              RA_IGNORED);
     if (ret < 0) {
@@ -1778,6 +1760,8 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 
     qemu_mutex_lock_iothread();
 
+    cpu_synchronize_state(CPU(cpu));
+
     switch (run->exit_reason) {
         case KVM_EXIT_S390_SIEIC:
             ret = handle_intercept(cpu);
-- 
2.14.3


Re: [Qemu-devel] [PATCH RFC] s390x/kvm: call cpu_synchronize_state() on every kvm_arch_handle_exit()
Posted by David Hildenbrand 6 years ago
On 26.03.2018 11:20, David Hildenbrand wrote:
> Manually having to use cpu_synchronize_state() is error prone. I don't
> think that the performance impact is that huge if we simply synchronize
> the state on every kvm_arch_handle_exit() call. This makes the code
> easier to maintain.
> 
> We now also call it (although not neded) for
> - KVM_EXIT_S390_RESET -> s390_reipl_request()
> - KVM_EXIT_DEBUG -> kvm_arch_handle_debug_exit()
> - unmanagable/unimplemented intercepts
> - ICPT_WAITPSW -> s390_handle_wait() -> cpu gets halted

Just noticed that this one is actually also already called :)

> - ICPT_CPU_STOP -> do_stop_interrupt() -> cpu gets halted
> - Scenarios where we inject an operation exception
> - handle_sigp() on the source CPU

And this one, too.

> - handle_stsi()

-- 

Thanks,

David / dhildenb

Re: [Qemu-devel] [PATCH RFC] s390x/kvm: call cpu_synchronize_state() on every kvm_arch_handle_exit()
Posted by Christian Borntraeger 5 years, 11 months ago

On 03/26/2018 11:20 AM, David Hildenbrand wrote:
> Manually having to use cpu_synchronize_state() is error prone. I don't
> think that the performance impact is that huge if we simply synchronize
> the state on every kvm_arch_handle_exit() call. This makes the code
> easier to maintain.
> 
> We now also call it (although not neded) for
> - KVM_EXIT_S390_RESET -> s390_reipl_request()
> - KVM_EXIT_DEBUG -> kvm_arch_handle_debug_exit()
> - unmanagable/unimplemented intercepts
> - ICPT_WAITPSW -> s390_handle_wait() -> cpu gets halted
> - ICPT_CPU_STOP -> do_stop_interrupt() -> cpu gets halted
> - Scenarios where we inject an operation exception
> - handle_sigp() on the source CPU
> - handle_stsi()
> 
> I don't think any of these are performance critical. Especially as we
> have all information directly contained in kvm_run, there are no
> additional IOCTLs to issue on modern kernels.

We had other issues in the past in other (common code) places. For example
see

commit 79ca7a1b898eb97c4192f3c78027a0f20485e7b4
Author:     Christian Borntraeger <borntraeger@de.ibm.com>
AuthorDate: Tue Mar 7 15:19:08 2017 +0100
Commit:     Paolo Bonzini <pbonzini@redhat.com>
CommitDate: Tue Mar 14 13:26:36 2017 +0100

    exec: add cpu_synchronize_state to cpu_memory_rw_debug

so we might consider going even further.....But this will be tricky.


FWIW, I think your patch even fixes a bug:

--- snip ----
static int handle_diag(S390CPU *cpu, struct kvm_run *run, uint32_t ipb)
{
    int r = 0;
    uint16_t func_code;

    /*
     * For any diagnose call we support, bits 48-63 of the resulting
     * address specify the function code; the remainder is ignored.
     */
    func_code = decode_basedisp_rs(&cpu->env, ipb, NULL) & DIAG_KVM_CODE_MASK;

---->
	static inline hwaddr decode_basedisp_s(CPUS390XState *env, uint32_t ipb,
	                                       uint8_t *ar)
	{
	    hwaddr addr = 0;
	    uint8_t reg;
	    
	    reg = ipb >> 28;
	    if (reg > 0) {
	        addr = env->regs[reg];

----> we do the sync_regs after this in the diag handler!
So currently we only handle the case with base reg == 0 correctly.
So
diag x,y,0x500(0) 
works


but things like
lghi 1,0x500
diag x,y,0(1)

not unless I miss something.


[...]
> @@ -1778,6 +1760,8 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
> 
>      qemu_mutex_lock_iothread();
> 
> +    cpu_synchronize_state(CPU(cpu));
> +
>      switch (run->exit_reason) {
>          case KVM_EXIT_S390_SIEIC:
>              ret = handle_intercept(cpu);
> 

Does it make sense to do this hunk NOW for 2.12 (cc stable)
and queue your full patch for 2.13?


Re: [Qemu-devel] [PATCH RFC] s390x/kvm: call cpu_synchronize_state() on every kvm_arch_handle_exit()
Posted by David Hildenbrand 5 years, 11 months ago
On 05.04.2018 09:53, Christian Borntraeger wrote:
> 
> 
> On 03/26/2018 11:20 AM, David Hildenbrand wrote:
>> Manually having to use cpu_synchronize_state() is error prone. I don't
>> think that the performance impact is that huge if we simply synchronize
>> the state on every kvm_arch_handle_exit() call. This makes the code
>> easier to maintain.
>>
>> We now also call it (although not neded) for
>> - KVM_EXIT_S390_RESET -> s390_reipl_request()
>> - KVM_EXIT_DEBUG -> kvm_arch_handle_debug_exit()
>> - unmanagable/unimplemented intercepts
>> - ICPT_WAITPSW -> s390_handle_wait() -> cpu gets halted
>> - ICPT_CPU_STOP -> do_stop_interrupt() -> cpu gets halted
>> - Scenarios where we inject an operation exception
>> - handle_sigp() on the source CPU
>> - handle_stsi()
>>
>> I don't think any of these are performance critical. Especially as we
>> have all information directly contained in kvm_run, there are no
>> additional IOCTLs to issue on modern kernels.
> 
> We had other issues in the past in other (common code) places. For example
> see
> 
> commit 79ca7a1b898eb97c4192f3c78027a0f20485e7b4
> Author:     Christian Borntraeger <borntraeger@de.ibm.com>
> AuthorDate: Tue Mar 7 15:19:08 2017 +0100
> Commit:     Paolo Bonzini <pbonzini@redhat.com>
> CommitDate: Tue Mar 14 13:26:36 2017 +0100
> 
>     exec: add cpu_synchronize_state to cpu_memory_rw_debug
> 
> so we might consider going even further.....But this will be tricky.
> 
> 
> FWIW, I think your patch even fixes a bug:
> 
> --- snip ----
> static int handle_diag(S390CPU *cpu, struct kvm_run *run, uint32_t ipb)
> {
>     int r = 0;
>     uint16_t func_code;
> 
>     /*
>      * For any diagnose call we support, bits 48-63 of the resulting
>      * address specify the function code; the remainder is ignored.
>      */
>     func_code = decode_basedisp_rs(&cpu->env, ipb, NULL) & DIAG_KVM_CODE_MASK;
> 
> ---->
> 	static inline hwaddr decode_basedisp_s(CPUS390XState *env, uint32_t ipb,
> 	                                       uint8_t *ar)
> 	{
> 	    hwaddr addr = 0;
> 	    uint8_t reg;
> 	    
> 	    reg = ipb >> 28;
> 	    if (reg > 0) {
> 	        addr = env->regs[reg];
> 
> ----> we do the sync_regs after this in the diag handler!
> So currently we only handle the case with base reg == 0 correctly.
> So
> diag x,y,0x500(0) 
> works
> 
> 
> but things like
> lghi 1,0x500
> diag x,y,0(1)
> 
> not unless I miss something.
> 
> 
> [...]
>> @@ -1778,6 +1760,8 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
>>
>>      qemu_mutex_lock_iothread();
>>
>> +    cpu_synchronize_state(CPU(cpu));
>> +
>>      switch (run->exit_reason) {
>>          case KVM_EXIT_S390_SIEIC:
>>              ret = handle_intercept(cpu);
>>
> 
> Does it make sense to do this hunk NOW for 2.12 (cc stable)
> and queue your full patch for 2.13?
> 

I think so - Conny?

-- 

Thanks,

David / dhildenb

Re: [Qemu-devel] [PATCH RFC] s390x/kvm: call cpu_synchronize_state() on every kvm_arch_handle_exit()
Posted by Cornelia Huck 5 years, 11 months ago
On Thu, 5 Apr 2018 09:53:45 +0200
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> On 03/26/2018 11:20 AM, David Hildenbrand wrote:

> FWIW, I think your patch even fixes a bug:
> 
> --- snip ----
> static int handle_diag(S390CPU *cpu, struct kvm_run *run, uint32_t ipb)
> {
>     int r = 0;
>     uint16_t func_code;
> 
>     /*
>      * For any diagnose call we support, bits 48-63 of the resulting
>      * address specify the function code; the remainder is ignored.
>      */
>     func_code = decode_basedisp_rs(&cpu->env, ipb, NULL) & DIAG_KVM_CODE_MASK;
> 
> ---->  
> 	static inline hwaddr decode_basedisp_s(CPUS390XState *env, uint32_t ipb,
> 	                                       uint8_t *ar)
> 	{
> 	    hwaddr addr = 0;
> 	    uint8_t reg;
> 	    
> 	    reg = ipb >> 28;
> 	    if (reg > 0) {
> 	        addr = env->regs[reg];
> 
> ----> we do the sync_regs after this in the diag handler!  
> So currently we only handle the case with base reg == 0 correctly.
> So
> diag x,y,0x500(0) 
> works
> 
> 
> but things like
> lghi 1,0x500
> diag x,y,0(1)
> 
> not unless I miss something.

I think you are right.

> 
> 
> [...]
> > @@ -1778,6 +1760,8 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
> > 
> >      qemu_mutex_lock_iothread();
> > 
> > +    cpu_synchronize_state(CPU(cpu));
> > +
> >      switch (run->exit_reason) {
> >          case KVM_EXIT_S390_SIEIC:
> >              ret = handle_intercept(cpu);
> >   
> 
> Does it make sense to do this hunk NOW for 2.12 (cc stable)
> and queue your full patch for 2.13?
> 

I'd prefer a cpu_synchronize_state() at the start of handle_diag() and
a respin of this patch for 2.13, but that should be fine as well.

I'll queue a proper patch for 2.12-rc.

Re: [Qemu-devel] [PATCH RFC] s390x/kvm: call cpu_synchronize_state() on every kvm_arch_handle_exit()
Posted by Thomas Huth 5 years, 11 months ago
On 05.04.2018 09:53, Christian Borntraeger wrote:
> 
> 
> On 03/26/2018 11:20 AM, David Hildenbrand wrote:
>> Manually having to use cpu_synchronize_state() is error prone. I don't
>> think that the performance impact is that huge if we simply synchronize
>> the state on every kvm_arch_handle_exit() call. This makes the code
>> easier to maintain.
>>
>> We now also call it (although not neded) for
>> - KVM_EXIT_S390_RESET -> s390_reipl_request()
>> - KVM_EXIT_DEBUG -> kvm_arch_handle_debug_exit()
>> - unmanagable/unimplemented intercepts
>> - ICPT_WAITPSW -> s390_handle_wait() -> cpu gets halted
>> - ICPT_CPU_STOP -> do_stop_interrupt() -> cpu gets halted
>> - Scenarios where we inject an operation exception
>> - handle_sigp() on the source CPU
>> - handle_stsi()
>>
>> I don't think any of these are performance critical. Especially as we
>> have all information directly contained in kvm_run, there are no
>> additional IOCTLs to issue on modern kernels.
> 
> We had other issues in the past in other (common code) places. For example
> see
> 
> commit 79ca7a1b898eb97c4192f3c78027a0f20485e7b4
> Author:     Christian Borntraeger <borntraeger@de.ibm.com>
> AuthorDate: Tue Mar 7 15:19:08 2017 +0100
> Commit:     Paolo Bonzini <pbonzini@redhat.com>
> CommitDate: Tue Mar 14 13:26:36 2017 +0100
> 
>     exec: add cpu_synchronize_state to cpu_memory_rw_debug
> 
> so we might consider going even further.....But this will be tricky.
> 
> 
> FWIW, I think your patch even fixes a bug:
> 
> --- snip ----
> static int handle_diag(S390CPU *cpu, struct kvm_run *run, uint32_t ipb)
> {
>     int r = 0;
>     uint16_t func_code;
> 
>     /*
>      * For any diagnose call we support, bits 48-63 of the resulting
>      * address specify the function code; the remainder is ignored.
>      */
>     func_code = decode_basedisp_rs(&cpu->env, ipb, NULL) & DIAG_KVM_CODE_MASK;
> 
> ---->
> 	static inline hwaddr decode_basedisp_s(CPUS390XState *env, uint32_t ipb,
> 	                                       uint8_t *ar)
> 	{
> 	    hwaddr addr = 0;
> 	    uint8_t reg;
> 	    
> 	    reg = ipb >> 28;
> 	    if (reg > 0) {
> 	        addr = env->regs[reg];
> 
> ----> we do the sync_regs after this in the diag handler!
> So currently we only handle the case with base reg == 0 correctly.
> So
> diag x,y,0x500(0) 
> works
> 
> 
> but things like
> lghi 1,0x500
> diag x,y,0(1)
> 
> not unless I miss something.

FWIW: Sounds like a good idea for a new kvm-unit-test... any volunteers?

 Thomas

Re: [Qemu-devel] [qemu-s390x] [PATCH RFC] s390x/kvm: call cpu_synchronize_state() on every kvm_arch_handle_exit()
Posted by Christian Borntraeger 5 years, 11 months ago

On 04/05/2018 10:19 AM, Thomas Huth wrote:
> On 05.04.2018 09:53, Christian Borntraeger wrote:

>> So currently we only handle the case with base reg == 0 correctly.
>> So
>> diag x,y,0x500(0) 
>> works
>>
>>
>> but things like
>> lghi 1,0x500
>> diag x,y,0(1)
>>
>> not unless I miss something.
> 
> FWIW: Sounds like a good idea for a new kvm-unit-test... any volunteers?

It will require some special handling in the test as eventfd will handle diag
in the kvm kernel module most of the time. So such a test must have 2 pathes
with and without eventfd. As a virtio device we could simply use null-co or 
something like that.