In a similar fashion as the previous patch, let's move the
handling of ibm,associativity-lookup-arrays from spapr.c to
spapr_numa.c. A spapr_numa_write_assoc_lookup_arrays() helper was
created, and spapr_dt_dynamic_reconfiguration_memory() can now
use it to advertise the lookup-arrays.

Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
---
 hw/ppc/spapr.c              | 25 ++----------------------
 hw/ppc/spapr_numa.c         | 39 +++++++++++++++++++++++++++++++++++++
 include/hw/ppc/spapr_numa.h |  2 ++
 3 files changed, 43 insertions(+), 23 deletions(-)

diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 172f965fe0..65d2ccd578 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -535,13 +535,10 @@ static int spapr_dt_dynamic_reconfiguration_memory(SpaprMachineState *spapr,
                                                    void *fdt)
 {
     MachineState *machine = MACHINE(spapr);
-    int nb_numa_nodes = machine->numa_state->num_nodes;
-    int ret, i, offset;
+    int ret, offset;
     uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
     uint32_t prop_lmb_size[] = {cpu_to_be32(lmb_size >> 32),
                                 cpu_to_be32(lmb_size & 0xffffffff)};
-    uint32_t *int_buf, *cur_index, buf_len;
-    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
     MemoryDeviceInfoList *dimms = NULL;
 
     /*
@@ -582,25 +579,7 @@ static int spapr_dt_dynamic_reconfiguration_memory(SpaprMachineState *spapr,
         return ret;
     }
 
-    /* ibm,associativity-lookup-arrays */
-    buf_len = (nr_nodes * 4 + 2) * sizeof(uint32_t);
-    cur_index = int_buf = g_malloc0(buf_len);
-    int_buf[0] = cpu_to_be32(nr_nodes);
-    int_buf[1] = cpu_to_be32(4); /* Number of entries per associativity list */
-    cur_index += 2;
-    for (i = 0; i < nr_nodes; i++) {
-        uint32_t associativity[] = {
-            cpu_to_be32(0x0),
-            cpu_to_be32(0x0),
-            cpu_to_be32(0x0),
-            cpu_to_be32(i)
-        };
-        memcpy(cur_index, associativity, sizeof(associativity));
-        cur_index += 4;
-    }
-    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
-            (cur_index - int_buf) * sizeof(uint32_t));
-    g_free(int_buf);
+    ret = spapr_numa_write_assoc_lookup_arrays(spapr, fdt, offset);
 
     return ret;
 }
diff --git a/hw/ppc/spapr_numa.c b/hw/ppc/spapr_numa.c
index b8882d209e..9eb4bdbe80 100644
--- a/hw/ppc/spapr_numa.c
+++ b/hw/ppc/spapr_numa.c
@@ -75,6 +75,45 @@ int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt,
                        vcpu_assoc, sizeof(vcpu_assoc));
 }
 
+
+int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt,
+                                         int offset)
+{
+    MachineState *machine = MACHINE(spapr);
+    SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
+    int nb_numa_nodes = machine->numa_state->num_nodes;
+    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
+    uint32_t *int_buf, *cur_index, buf_len;
+    int ret, i, j;
+
+    /* ibm,associativity-lookup-arrays */
+    buf_len = (nr_nodes * MAX_DISTANCE_REF_POINTS + 2) * sizeof(uint32_t);
+    cur_index = int_buf = g_malloc0(buf_len);
+    int_buf[0] = cpu_to_be32(nr_nodes);
+     /* Number of entries per associativity list */
+    int_buf[1] = cpu_to_be32(MAX_DISTANCE_REF_POINTS);
+    cur_index += 2;
+    for (i = 0; i < nr_nodes; i++) {
+        /*
+         * For the lookup-array we use the ibm,associativity array,
+         * from numa_assoc_array. without the first element (size).
+         */
+        uint32_t associativity[MAX_DISTANCE_REF_POINTS];
+
+        for (j = 0; j < MAX_DISTANCE_REF_POINTS; j++) {
+            associativity[j] = smc->numa_assoc_array[i][j + 1];
+        }
+
+        memcpy(cur_index, associativity, sizeof(associativity));
+        cur_index += 4;
+    }
+    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
+                      (cur_index - int_buf) * sizeof(uint32_t));
+    g_free(int_buf);
+
+    return ret;
+}
+
 /*
  * Helper that writes ibm,associativity-reference-points and
  * max-associativity-domains in the RTAS pointed by @rtas
diff --git a/include/hw/ppc/spapr_numa.h b/include/hw/ppc/spapr_numa.h
index f92fb4f28a..f6127501a6 100644
--- a/include/hw/ppc/spapr_numa.h
+++ b/include/hw/ppc/spapr_numa.h
@@ -22,6 +22,8 @@ void spapr_numa_write_associativity_dt(SpaprMachineState *spapr, void *fdt,
                                        int offset, int nodeid);
 int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt,
                             int offset, PowerPCCPU *cpu);
+int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt,
+                                         int offset);
 
 
 #endif /* HW_SPAPR_NUMA_H */
-- 
2.26.2

On Tue, Sep 01, 2020 at 09:56:43AM -0300, Daniel Henrique Barboza wrote:
> In a similar fashion as the previous patch, let's move the
> handling of ibm,associativity-lookup-arrays from spapr.c to
> spapr_numa.c. A spapr_numa_write_assoc_lookup_arrays() helper was
> created, and spapr_dt_dynamic_reconfiguration_memory() can now
> use it to advertise the lookup-arrays.
> 
> Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
> ---
>  hw/ppc/spapr.c              | 25 ++----------------------
>  hw/ppc/spapr_numa.c         | 39 +++++++++++++++++++++++++++++++++++++
>  include/hw/ppc/spapr_numa.h |  2 ++
>  3 files changed, 43 insertions(+), 23 deletions(-)
> 
> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> index 172f965fe0..65d2ccd578 100644
> --- a/hw/ppc/spapr.c
> +++ b/hw/ppc/spapr.c
> @@ -535,13 +535,10 @@ static int spapr_dt_dynamic_reconfiguration_memory(SpaprMachineState *spapr,
>                                                     void *fdt)
>  {
>      MachineState *machine = MACHINE(spapr);
> -    int nb_numa_nodes = machine->numa_state->num_nodes;
> -    int ret, i, offset;
> +    int ret, offset;
>      uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
>      uint32_t prop_lmb_size[] = {cpu_to_be32(lmb_size >> 32),
>                                  cpu_to_be32(lmb_size & 0xffffffff)};
> -    uint32_t *int_buf, *cur_index, buf_len;
> -    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
>      MemoryDeviceInfoList *dimms = NULL;
>  
>      /*
> @@ -582,25 +579,7 @@ static int spapr_dt_dynamic_reconfiguration_memory(SpaprMachineState *spapr,
>          return ret;
>      }
>  
> -    /* ibm,associativity-lookup-arrays */
> -    buf_len = (nr_nodes * 4 + 2) * sizeof(uint32_t);
> -    cur_index = int_buf = g_malloc0(buf_len);
> -    int_buf[0] = cpu_to_be32(nr_nodes);
> -    int_buf[1] = cpu_to_be32(4); /* Number of entries per associativity list */
> -    cur_index += 2;
> -    for (i = 0; i < nr_nodes; i++) {
> -        uint32_t associativity[] = {
> -            cpu_to_be32(0x0),
> -            cpu_to_be32(0x0),
> -            cpu_to_be32(0x0),
> -            cpu_to_be32(i)
> -        };
> -        memcpy(cur_index, associativity, sizeof(associativity));
> -        cur_index += 4;
> -    }
> -    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
> -            (cur_index - int_buf) * sizeof(uint32_t));
> -    g_free(int_buf);
> +    ret = spapr_numa_write_assoc_lookup_arrays(spapr, fdt, offset);
>  
>      return ret;
>  }
> diff --git a/hw/ppc/spapr_numa.c b/hw/ppc/spapr_numa.c
> index b8882d209e..9eb4bdbe80 100644
> --- a/hw/ppc/spapr_numa.c
> +++ b/hw/ppc/spapr_numa.c
> @@ -75,6 +75,45 @@ int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt,
>                         vcpu_assoc, sizeof(vcpu_assoc));
>  }
>  
> +
> +int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt,
> +                                         int offset)
> +{
> +    MachineState *machine = MACHINE(spapr);
> +    SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
> +    int nb_numa_nodes = machine->numa_state->num_nodes;
> +    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
> +    uint32_t *int_buf, *cur_index, buf_len;
> +    int ret, i, j;
> +
> +    /* ibm,associativity-lookup-arrays */
> +    buf_len = (nr_nodes * MAX_DISTANCE_REF_POINTS + 2) * sizeof(uint32_t);
> +    cur_index = int_buf = g_malloc0(buf_len);
> +    int_buf[0] = cpu_to_be32(nr_nodes);
> +     /* Number of entries per associativity list */
> +    int_buf[1] = cpu_to_be32(MAX_DISTANCE_REF_POINTS);
> +    cur_index += 2;
> +    for (i = 0; i < nr_nodes; i++) {
> +        /*
> +         * For the lookup-array we use the ibm,associativity array,
> +         * from numa_assoc_array. without the first element (size).
> +         */
> +        uint32_t associativity[MAX_DISTANCE_REF_POINTS];
> +
> +        for (j = 0; j < MAX_DISTANCE_REF_POINTS; j++) {
> +            associativity[j] = smc->numa_assoc_array[i][j + 1];
> +        }
> +
> +        memcpy(cur_index, associativity, sizeof(associativity));

AFAICT, you could just use a single memcpy() to copy from the
numa_assoc_array() into the property here, rather than using a loop
and temporary array.

> +        cur_index += 4;

Shouldn't this be  += MAX_DISTANCE_REF_POINTS?

> +    }
> +    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
> +                      (cur_index - int_buf) * sizeof(uint32_t));
> +    g_free(int_buf);
> +
> +    return ret;
> +}
> +
>  /*
>   * Helper that writes ibm,associativity-reference-points and
>   * max-associativity-domains in the RTAS pointed by @rtas
> diff --git a/include/hw/ppc/spapr_numa.h b/include/hw/ppc/spapr_numa.h
> index f92fb4f28a..f6127501a6 100644
> --- a/include/hw/ppc/spapr_numa.h
> +++ b/include/hw/ppc/spapr_numa.h
> @@ -22,6 +22,8 @@ void spapr_numa_write_associativity_dt(SpaprMachineState *spapr, void *fdt,
>                                         int offset, int nodeid);
>  int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt,
>                              int offset, PowerPCCPU *cpu);
> +int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt,
> +                                         int offset);
>  
>  
>  #endif /* HW_SPAPR_NUMA_H */

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

On 9/2/20 10:34 PM, David Gibson wrote:
> On Tue, Sep 01, 2020 at 09:56:43AM -0300, Daniel Henrique Barboza wrote:
>> In a similar fashion as the previous patch, let's move the
>> handling of ibm,associativity-lookup-arrays from spapr.c to
>> spapr_numa.c. A spapr_numa_write_assoc_lookup_arrays() helper was
>> created, and spapr_dt_dynamic_reconfiguration_memory() can now
>> use it to advertise the lookup-arrays.
>>
>> Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
>> ---
>>   hw/ppc/spapr.c              | 25 ++----------------------
>>   hw/ppc/spapr_numa.c         | 39 +++++++++++++++++++++++++++++++++++++
>>   include/hw/ppc/spapr_numa.h |  2 ++
>>   3 files changed, 43 insertions(+), 23 deletions(-)
>>
>> diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
>> index 172f965fe0..65d2ccd578 100644
>> --- a/hw/ppc/spapr.c
>> +++ b/hw/ppc/spapr.c
>> @@ -535,13 +535,10 @@ static int spapr_dt_dynamic_reconfiguration_memory(SpaprMachineState *spapr,
>>                                                      void *fdt)
>>   {
>>       MachineState *machine = MACHINE(spapr);
>> -    int nb_numa_nodes = machine->numa_state->num_nodes;
>> -    int ret, i, offset;
>> +    int ret, offset;
>>       uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
>>       uint32_t prop_lmb_size[] = {cpu_to_be32(lmb_size >> 32),
>>                                   cpu_to_be32(lmb_size & 0xffffffff)};
>> -    uint32_t *int_buf, *cur_index, buf_len;
>> -    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
>>       MemoryDeviceInfoList *dimms = NULL;
>>   
>>       /*
>> @@ -582,25 +579,7 @@ static int spapr_dt_dynamic_reconfiguration_memory(SpaprMachineState *spapr,
>>           return ret;
>>       }
>>   
>> -    /* ibm,associativity-lookup-arrays */
>> -    buf_len = (nr_nodes * 4 + 2) * sizeof(uint32_t);
>> -    cur_index = int_buf = g_malloc0(buf_len);
>> -    int_buf[0] = cpu_to_be32(nr_nodes);
>> -    int_buf[1] = cpu_to_be32(4); /* Number of entries per associativity list */
>> -    cur_index += 2;
>> -    for (i = 0; i < nr_nodes; i++) {
>> -        uint32_t associativity[] = {
>> -            cpu_to_be32(0x0),
>> -            cpu_to_be32(0x0),
>> -            cpu_to_be32(0x0),
>> -            cpu_to_be32(i)
>> -        };
>> -        memcpy(cur_index, associativity, sizeof(associativity));
>> -        cur_index += 4;
>> -    }
>> -    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
>> -            (cur_index - int_buf) * sizeof(uint32_t));
>> -    g_free(int_buf);
>> +    ret = spapr_numa_write_assoc_lookup_arrays(spapr, fdt, offset);
>>   
>>       return ret;
>>   }
>> diff --git a/hw/ppc/spapr_numa.c b/hw/ppc/spapr_numa.c
>> index b8882d209e..9eb4bdbe80 100644
>> --- a/hw/ppc/spapr_numa.c
>> +++ b/hw/ppc/spapr_numa.c
>> @@ -75,6 +75,45 @@ int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt,
>>                          vcpu_assoc, sizeof(vcpu_assoc));
>>   }
>>   
>> +
>> +int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt,
>> +                                         int offset)
>> +{
>> +    MachineState *machine = MACHINE(spapr);
>> +    SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
>> +    int nb_numa_nodes = machine->numa_state->num_nodes;
>> +    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
>> +    uint32_t *int_buf, *cur_index, buf_len;
>> +    int ret, i, j;
>> +
>> +    /* ibm,associativity-lookup-arrays */
>> +    buf_len = (nr_nodes * MAX_DISTANCE_REF_POINTS + 2) * sizeof(uint32_t);
>> +    cur_index = int_buf = g_malloc0(buf_len);
>> +    int_buf[0] = cpu_to_be32(nr_nodes);
>> +     /* Number of entries per associativity list */
>> +    int_buf[1] = cpu_to_be32(MAX_DISTANCE_REF_POINTS);
>> +    cur_index += 2;
>> +    for (i = 0; i < nr_nodes; i++) {
>> +        /*
>> +         * For the lookup-array we use the ibm,associativity array,
>> +         * from numa_assoc_array. without the first element (size).
>> +         */
>> +        uint32_t associativity[MAX_DISTANCE_REF_POINTS];
>> +
>> +        for (j = 0; j < MAX_DISTANCE_REF_POINTS; j++) {
>> +            associativity[j] = smc->numa_assoc_array[i][j + 1];
>> +        }
>> +
>> +        memcpy(cur_index, associativity, sizeof(associativity));
> 
> AFAICT, you could just use a single memcpy() to copy from the
> numa_assoc_array() into the property here, rather than using a loop
> and temporary array.

I remember that I was having some weird problems with memcpy() and
numa_assoc_array and this is how I got around it. I'll try to sort it
out again.

> 
>> +        cur_index += 4;
> 
> Shouldn't this be  += MAX_DISTANCE_REF_POINTS?


Yeah it should. Good catch.

> 
>> +    }
>> +    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
>> +                      (cur_index - int_buf) * sizeof(uint32_t));
>> +    g_free(int_buf);
>> +
>> +    return ret;
>> +}
>> +
>>   /*
>>    * Helper that writes ibm,associativity-reference-points and
>>    * max-associativity-domains in the RTAS pointed by @rtas
>> diff --git a/include/hw/ppc/spapr_numa.h b/include/hw/ppc/spapr_numa.h
>> index f92fb4f28a..f6127501a6 100644
>> --- a/include/hw/ppc/spapr_numa.h
>> +++ b/include/hw/ppc/spapr_numa.h
>> @@ -22,6 +22,8 @@ void spapr_numa_write_associativity_dt(SpaprMachineState *spapr, void *fdt,
>>                                          int offset, int nodeid);
>>   int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt,
>>                               int offset, PowerPCCPU *cpu);
>> +int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt,
>> +                                         int offset);
>>   
>>   
>>   #endif /* HW_SPAPR_NUMA_H */
>