Provide the interface to list the assignment states of all the resctrl
groups in mbm_cntr_assign mode.
Example:
$mount -t resctrl resctrl /sys/fs/resctrl/
$cat /sys/fs/resctrl/info/L3_MON/mbm_assign_control
//0=tl;1=tl;
List follows the following format:
"<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
Format for specific type of groups:
- Default CTRL_MON group:
"//<domain_id>=<flags>"
- Non-default CTRL_MON group:
"<CTRL_MON group>//<domain_id>=<flags>"
- Child MON group of default CTRL_MON group:
"/<MON group>/<domain_id>=<flags>"
- Child MON group of non-default CTRL_MON group:
"<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
Flags can be one of the following:
t MBM total event is enabled
l MBM local event is enabled
tl Both total and local MBM events are enabled
_ None of the MBM events are enabled
Signed-off-by: Babu Moger <babu.moger@amd.com>
---
v8: Moved resctrl_mbm_event_assigned() in here as it is first used here.
Moved rdt_last_cmd_clear() before making any call.
Updated the commit log.
Corrected the doc format.
v7: Renamed the interface name from 'mbm_control' to 'mbm_assign_control'
to match 'mbm_assign_mode'.
Removed Arch references from FS code.
Added rdt_last_cmd_clear() before the command processing.
Added rdtgroup_mutex before all the calls.
Removed references of ABMC from FS code.
v6: The domain specific assignment can be determined looking at mbm_cntr_map.
Removed rdtgroup_abmc_dom_cfg() and rdtgroup_abmc_dom_state().
Removed the switch statement for the domain_state detection.
Determined the flags incremently.
Removed special handling of default group while printing..
v5: Replaced "assignment flags" with "flags".
Changes related to mon structure.
Changes related renaming the interface from mbm_assign_control to
mbm_control.
v4: Added functionality to query domain specific assigment in.
rdtgroup_abmc_dom_state().
v3: New patch.
Addresses the feedback to provide the global assignment interface.
https://lore.kernel.org/lkml/c73f444b-83a1-4e9a-95d3-54c5165ee782@intel.com/
---
Documentation/arch/x86/resctrl.rst | 44 +++++++++++++++
arch/x86/kernel/cpu/resctrl/monitor.c | 1 +
arch/x86/kernel/cpu/resctrl/rdtgroup.c | 76 ++++++++++++++++++++++++++
3 files changed, 121 insertions(+)
diff --git a/Documentation/arch/x86/resctrl.rst b/Documentation/arch/x86/resctrl.rst
index d9574078f735..b85d3bc3e301 100644
--- a/Documentation/arch/x86/resctrl.rst
+++ b/Documentation/arch/x86/resctrl.rst
@@ -310,6 +310,50 @@ with the following files:
The number of monitoring counters available for assignment when the
architecture supports mbm_cntr_assign mode.
+"mbm_assign_control":
+ Reports the resctrl group and monitor status of each group.
+
+ List follows the following format:
+ "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
+
+ Format for specific type of groups:
+
+ * Default CTRL_MON group:
+ "//<domain_id>=<flags>"
+
+ * Non-default CTRL_MON group:
+ "<CTRL_MON group>//<domain_id>=<flags>"
+
+ * Child MON group of default CTRL_MON group:
+ "/<MON group>/<domain_id>=<flags>"
+
+ * Child MON group of non-default CTRL_MON group:
+ "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
+
+ Flags can be one of the following:
+ ::
+
+ t MBM total event is assigned.
+ l MBM local event is assigned.
+ tl Both total and local MBM events are assigned.
+ _ None of the MBM events are assigned.
+
+ Examples:
+ ::
+
+ # mkdir /sys/fs/resctrl/mon_groups/child_default_mon_grp
+ # mkdir /sys/fs/resctrl/non_default_ctrl_mon_grp
+ # mkdir /sys/fs/resctrl/non_default_ctrl_mon_grp/mon_groups/child_non_default_mon_grp
+
+ # cat /sys/fs/resctrl/info/L3_MON/mbm_assign_control
+ non_default_ctrl_mon_grp//0=tl;1=tl;
+ non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=tl;
+ //0=tl;1=tl;
+ /child_default_mon_grp/0=tl;1=tl;
+
+ There are four resctrl groups. All the groups have total and local MBM events
+ assigned on domain 0 and 1.
+
"max_threshold_occupancy":
Read/write file provides the largest value (in
bytes) at which a previously used LLC_occupancy
diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
index 395d99984893..fa7c77935080 100644
--- a/arch/x86/kernel/cpu/resctrl/monitor.c
+++ b/arch/x86/kernel/cpu/resctrl/monitor.c
@@ -1269,6 +1269,7 @@ int __init rdt_get_mon_l3_config(struct rdt_resource *r)
r->mon.num_mbm_cntrs = (ebx & GENMASK(15, 0)) + 1;
resctrl_file_fflags_init("num_mbm_cntrs", RFTYPE_MON_INFO);
hw_res->mbm_cntr_assign_enabled = true;
+ resctrl_file_fflags_init("mbm_assign_control", RFTYPE_MON_INFO);
}
}
diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
index cf2e0ad0e4f4..cf92ceb0f05e 100644
--- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
+++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
@@ -970,6 +970,76 @@ static int rdtgroup_num_mbm_cntrs_show(struct kernfs_open_file *of,
return 0;
}
+static bool resctrl_mbm_event_assigned(struct rdtgroup *rdtg,
+ struct rdt_mon_domain *d, u32 evtid)
+{
+ int index = MBM_EVENT_ARRAY_INDEX(evtid);
+ int cntr_id = rdtg->mon.cntr_id[index];
+
+ return cntr_id != MON_CNTR_UNSET && test_bit(cntr_id, d->mbm_cntr_map);
+}
+
+static char *rdtgroup_mon_state_to_str(struct rdtgroup *rdtgrp,
+ struct rdt_mon_domain *d, char *str)
+{
+ char *tmp = str;
+
+ /* Query the total and local event flags for the domain */
+ if (resctrl_mbm_event_assigned(rdtgrp, d, QOS_L3_MBM_TOTAL_EVENT_ID))
+ *tmp++ = 't';
+
+ if (resctrl_mbm_event_assigned(rdtgrp, d, QOS_L3_MBM_LOCAL_EVENT_ID))
+ *tmp++ = 'l';
+
+ if (tmp == str)
+ *tmp++ = '_';
+
+ *tmp = '\0';
+ return str;
+}
+
+static int rdtgroup_mbm_assign_control_show(struct kernfs_open_file *of,
+ struct seq_file *s, void *v)
+{
+ struct rdt_resource *r = of->kn->parent->priv;
+ struct rdt_mon_domain *dom;
+ struct rdtgroup *rdtg;
+ char str[10];
+
+ mutex_lock(&rdtgroup_mutex);
+ rdt_last_cmd_clear();
+
+ if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
+ rdt_last_cmd_puts("mbm_cntr_assign mode is not enabled\n");
+ mutex_unlock(&rdtgroup_mutex);
+ return -EINVAL;
+ }
+
+ list_for_each_entry(rdtg, &rdt_all_groups, rdtgroup_list) {
+ struct rdtgroup *crg;
+
+ seq_printf(s, "%s//", rdtg->kn->name);
+
+ list_for_each_entry(dom, &r->mon_domains, hdr.list)
+ seq_printf(s, "%d=%s;", dom->hdr.id,
+ rdtgroup_mon_state_to_str(rdtg, dom, str));
+ seq_putc(s, '\n');
+
+ list_for_each_entry(crg, &rdtg->mon.crdtgrp_list,
+ mon.crdtgrp_list) {
+ seq_printf(s, "%s/%s/", rdtg->kn->name, crg->kn->name);
+
+ list_for_each_entry(dom, &r->mon_domains, hdr.list)
+ seq_printf(s, "%d=%s;", dom->hdr.id,
+ rdtgroup_mon_state_to_str(crg, dom, str));
+ seq_putc(s, '\n');
+ }
+ }
+
+ mutex_unlock(&rdtgroup_mutex);
+ return 0;
+}
+
#ifdef CONFIG_PROC_CPU_RESCTRL
/*
@@ -2256,6 +2326,12 @@ static struct rftype res_common_files[] = {
.kf_ops = &rdtgroup_kf_single_ops,
.seq_show = rdtgroup_num_mbm_cntrs_show,
},
+ {
+ .name = "mbm_assign_control",
+ .mode = 0444,
+ .kf_ops = &rdtgroup_kf_single_ops,
+ .seq_show = rdtgroup_mbm_assign_control_show,
+ },
{
.name = "cpus_list",
.mode = 0644,
--
2.34.1
Hi Babu,
On 10/9/24 10:39 AM, Babu Moger wrote:
> Provide the interface to list the assignment states of all the resctrl
> groups in mbm_cntr_assign mode.
>
> Example:
> $mount -t resctrl resctrl /sys/fs/resctrl/
> $cat /sys/fs/resctrl/info/L3_MON/mbm_assign_control
> //0=tl;1=tl;
>
> List follows the following format:
>
> "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
>
> Format for specific type of groups:
>
> - Default CTRL_MON group:
> "//<domain_id>=<flags>"
>
> - Non-default CTRL_MON group:
> "<CTRL_MON group>//<domain_id>=<flags>"
>
> - Child MON group of default CTRL_MON group:
> "/<MON group>/<domain_id>=<flags>"
>
> - Child MON group of non-default CTRL_MON group:
> "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
>
> Flags can be one of the following:
> t MBM total event is enabled
> l MBM local event is enabled
> tl Both total and local MBM events are enabled
> _ None of the MBM events are enabled
>
> Signed-off-by: Babu Moger <babu.moger@amd.com>
> ---
> v8: Moved resctrl_mbm_event_assigned() in here as it is first used here.
> Moved rdt_last_cmd_clear() before making any call.
> Updated the commit log.
> Corrected the doc format.
>
> v7: Renamed the interface name from 'mbm_control' to 'mbm_assign_control'
> to match 'mbm_assign_mode'.
> Removed Arch references from FS code.
> Added rdt_last_cmd_clear() before the command processing.
> Added rdtgroup_mutex before all the calls.
> Removed references of ABMC from FS code.
>
> v6: The domain specific assignment can be determined looking at mbm_cntr_map.
> Removed rdtgroup_abmc_dom_cfg() and rdtgroup_abmc_dom_state().
> Removed the switch statement for the domain_state detection.
> Determined the flags incremently.
> Removed special handling of default group while printing..
>
> v5: Replaced "assignment flags" with "flags".
> Changes related to mon structure.
> Changes related renaming the interface from mbm_assign_control to
> mbm_control.
>
> v4: Added functionality to query domain specific assigment in.
> rdtgroup_abmc_dom_state().
>
> v3: New patch.
> Addresses the feedback to provide the global assignment interface.
> https://lore.kernel.org/lkml/c73f444b-83a1-4e9a-95d3-54c5165ee782@intel.com/
> ---
> Documentation/arch/x86/resctrl.rst | 44 +++++++++++++++
> arch/x86/kernel/cpu/resctrl/monitor.c | 1 +
> arch/x86/kernel/cpu/resctrl/rdtgroup.c | 76 ++++++++++++++++++++++++++
> 3 files changed, 121 insertions(+)
>
> diff --git a/Documentation/arch/x86/resctrl.rst b/Documentation/arch/x86/resctrl.rst
> index d9574078f735..b85d3bc3e301 100644
> --- a/Documentation/arch/x86/resctrl.rst
> +++ b/Documentation/arch/x86/resctrl.rst
> @@ -310,6 +310,50 @@ with the following files:
> The number of monitoring counters available for assignment when the
> architecture supports mbm_cntr_assign mode.
>
> +"mbm_assign_control":
> + Reports the resctrl group and monitor status of each group.
> +
> + List follows the following format:
> + "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
> +
> + Format for specific type of groups:
> +
> + * Default CTRL_MON group:
> + "//<domain_id>=<flags>"
> +
> + * Non-default CTRL_MON group:
> + "<CTRL_MON group>//<domain_id>=<flags>"
> +
> + * Child MON group of default CTRL_MON group:
> + "/<MON group>/<domain_id>=<flags>"
> +
> + * Child MON group of non-default CTRL_MON group:
> + "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
> +
> + Flags can be one of the following:
> + ::
> +
> + t MBM total event is assigned.
> + l MBM local event is assigned.
> + tl Both total and local MBM events are assigned.
> + _ None of the MBM events are assigned.
> +
> + Examples:
> + ::
> +
> + # mkdir /sys/fs/resctrl/mon_groups/child_default_mon_grp
> + # mkdir /sys/fs/resctrl/non_default_ctrl_mon_grp
> + # mkdir /sys/fs/resctrl/non_default_ctrl_mon_grp/mon_groups/child_non_default_mon_grp
> +
> + # cat /sys/fs/resctrl/info/L3_MON/mbm_assign_control
> + non_default_ctrl_mon_grp//0=tl;1=tl;
> + non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=tl;
> + //0=tl;1=tl;
> + /child_default_mon_grp/0=tl;1=tl;
> +
> + There are four resctrl groups. All the groups have total and local MBM events
> + assigned on domain 0 and 1.
> +
> "max_threshold_occupancy":
> Read/write file provides the largest value (in
> bytes) at which a previously used LLC_occupancy
> diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
> index 395d99984893..fa7c77935080 100644
> --- a/arch/x86/kernel/cpu/resctrl/monitor.c
> +++ b/arch/x86/kernel/cpu/resctrl/monitor.c
> @@ -1269,6 +1269,7 @@ int __init rdt_get_mon_l3_config(struct rdt_resource *r)
> r->mon.num_mbm_cntrs = (ebx & GENMASK(15, 0)) + 1;
> resctrl_file_fflags_init("num_mbm_cntrs", RFTYPE_MON_INFO);
> hw_res->mbm_cntr_assign_enabled = true;
> + resctrl_file_fflags_init("mbm_assign_control", RFTYPE_MON_INFO);
> }
> }
>
> diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
> index cf2e0ad0e4f4..cf92ceb0f05e 100644
> --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
> +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
> @@ -970,6 +970,76 @@ static int rdtgroup_num_mbm_cntrs_show(struct kernfs_open_file *of,
> return 0;
> }
>
> +static bool resctrl_mbm_event_assigned(struct rdtgroup *rdtg,
> + struct rdt_mon_domain *d, u32 evtid)
u32 -> enum resctrl_event_id ?
> +{
> + int index = MBM_EVENT_ARRAY_INDEX(evtid);
> + int cntr_id = rdtg->mon.cntr_id[index];
> +
> + return cntr_id != MON_CNTR_UNSET && test_bit(cntr_id, d->mbm_cntr_map);
> +}
> +
> +static char *rdtgroup_mon_state_to_str(struct rdtgroup *rdtgrp,
> + struct rdt_mon_domain *d, char *str)
> +{
> + char *tmp = str;
> +
> + /* Query the total and local event flags for the domain */
> + if (resctrl_mbm_event_assigned(rdtgrp, d, QOS_L3_MBM_TOTAL_EVENT_ID))
> + *tmp++ = 't';
> +
> + if (resctrl_mbm_event_assigned(rdtgrp, d, QOS_L3_MBM_LOCAL_EVENT_ID))
> + *tmp++ = 'l';
> +
> + if (tmp == str)
> + *tmp++ = '_';
> +
> + *tmp = '\0';
> + return str;
> +}
> +
Reinette
Hi Reinette,
On 10/15/24 22:40, Reinette Chatre wrote:
> Hi Babu,
>
> On 10/9/24 10:39 AM, Babu Moger wrote:
>> Provide the interface to list the assignment states of all the resctrl
>> groups in mbm_cntr_assign mode.
>>
>> Example:
>> $mount -t resctrl resctrl /sys/fs/resctrl/
>> $cat /sys/fs/resctrl/info/L3_MON/mbm_assign_control
>> //0=tl;1=tl;
>>
>> List follows the following format:
>>
>> "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
>>
>> Format for specific type of groups:
>>
>> - Default CTRL_MON group:
>> "//<domain_id>=<flags>"
>>
>> - Non-default CTRL_MON group:
>> "<CTRL_MON group>//<domain_id>=<flags>"
>>
>> - Child MON group of default CTRL_MON group:
>> "/<MON group>/<domain_id>=<flags>"
>>
>> - Child MON group of non-default CTRL_MON group:
>> "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
>>
>> Flags can be one of the following:
>> t MBM total event is enabled
>> l MBM local event is enabled
>> tl Both total and local MBM events are enabled
>> _ None of the MBM events are enabled
>>
>> Signed-off-by: Babu Moger <babu.moger@amd.com>
>> ---
>> v8: Moved resctrl_mbm_event_assigned() in here as it is first used here.
>> Moved rdt_last_cmd_clear() before making any call.
>> Updated the commit log.
>> Corrected the doc format.
>>
>> v7: Renamed the interface name from 'mbm_control' to 'mbm_assign_control'
>> to match 'mbm_assign_mode'.
>> Removed Arch references from FS code.
>> Added rdt_last_cmd_clear() before the command processing.
>> Added rdtgroup_mutex before all the calls.
>> Removed references of ABMC from FS code.
>>
>> v6: The domain specific assignment can be determined looking at mbm_cntr_map.
>> Removed rdtgroup_abmc_dom_cfg() and rdtgroup_abmc_dom_state().
>> Removed the switch statement for the domain_state detection.
>> Determined the flags incremently.
>> Removed special handling of default group while printing..
>>
>> v5: Replaced "assignment flags" with "flags".
>> Changes related to mon structure.
>> Changes related renaming the interface from mbm_assign_control to
>> mbm_control.
>>
>> v4: Added functionality to query domain specific assigment in.
>> rdtgroup_abmc_dom_state().
>>
>> v3: New patch.
>> Addresses the feedback to provide the global assignment interface.
>> https://lore.kernel.org/lkml/c73f444b-83a1-4e9a-95d3-54c5165ee782@intel.com/
>> ---
>> Documentation/arch/x86/resctrl.rst | 44 +++++++++++++++
>> arch/x86/kernel/cpu/resctrl/monitor.c | 1 +
>> arch/x86/kernel/cpu/resctrl/rdtgroup.c | 76 ++++++++++++++++++++++++++
>> 3 files changed, 121 insertions(+)
>>
>> diff --git a/Documentation/arch/x86/resctrl.rst b/Documentation/arch/x86/resctrl.rst
>> index d9574078f735..b85d3bc3e301 100644
>> --- a/Documentation/arch/x86/resctrl.rst
>> +++ b/Documentation/arch/x86/resctrl.rst
>> @@ -310,6 +310,50 @@ with the following files:
>> The number of monitoring counters available for assignment when the
>> architecture supports mbm_cntr_assign mode.
>>
>> +"mbm_assign_control":
>> + Reports the resctrl group and monitor status of each group.
>> +
>> + List follows the following format:
>> + "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
>> +
>> + Format for specific type of groups:
>> +
>> + * Default CTRL_MON group:
>> + "//<domain_id>=<flags>"
>> +
>> + * Non-default CTRL_MON group:
>> + "<CTRL_MON group>//<domain_id>=<flags>"
>> +
>> + * Child MON group of default CTRL_MON group:
>> + "/<MON group>/<domain_id>=<flags>"
>> +
>> + * Child MON group of non-default CTRL_MON group:
>> + "<CTRL_MON group>/<MON group>/<domain_id>=<flags>"
>> +
>> + Flags can be one of the following:
>> + ::
>> +
>> + t MBM total event is assigned.
>> + l MBM local event is assigned.
>> + tl Both total and local MBM events are assigned.
>> + _ None of the MBM events are assigned.
>> +
>> + Examples:
>> + ::
>> +
>> + # mkdir /sys/fs/resctrl/mon_groups/child_default_mon_grp
>> + # mkdir /sys/fs/resctrl/non_default_ctrl_mon_grp
>> + # mkdir /sys/fs/resctrl/non_default_ctrl_mon_grp/mon_groups/child_non_default_mon_grp
>> +
>> + # cat /sys/fs/resctrl/info/L3_MON/mbm_assign_control
>> + non_default_ctrl_mon_grp//0=tl;1=tl;
>> + non_default_ctrl_mon_grp/child_non_default_mon_grp/0=tl;1=tl;
>> + //0=tl;1=tl;
>> + /child_default_mon_grp/0=tl;1=tl;
>> +
>> + There are four resctrl groups. All the groups have total and local MBM events
>> + assigned on domain 0 and 1.
>> +
>> "max_threshold_occupancy":
>> Read/write file provides the largest value (in
>> bytes) at which a previously used LLC_occupancy
>> diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
>> index 395d99984893..fa7c77935080 100644
>> --- a/arch/x86/kernel/cpu/resctrl/monitor.c
>> +++ b/arch/x86/kernel/cpu/resctrl/monitor.c
>> @@ -1269,6 +1269,7 @@ int __init rdt_get_mon_l3_config(struct rdt_resource *r)
>> r->mon.num_mbm_cntrs = (ebx & GENMASK(15, 0)) + 1;
>> resctrl_file_fflags_init("num_mbm_cntrs", RFTYPE_MON_INFO);
>> hw_res->mbm_cntr_assign_enabled = true;
>> + resctrl_file_fflags_init("mbm_assign_control", RFTYPE_MON_INFO);
>> }
>> }
>>
>> diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
>> index cf2e0ad0e4f4..cf92ceb0f05e 100644
>> --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
>> +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
>> @@ -970,6 +970,76 @@ static int rdtgroup_num_mbm_cntrs_show(struct kernfs_open_file *of,
>> return 0;
>> }
>>
>> +static bool resctrl_mbm_event_assigned(struct rdtgroup *rdtg,
>> + struct rdt_mon_domain *d, u32 evtid)
>
> u32 -> enum resctrl_event_id ?
Sure.
>
>> +{
>> + int index = MBM_EVENT_ARRAY_INDEX(evtid);
>> + int cntr_id = rdtg->mon.cntr_id[index];
>> +
>> + return cntr_id != MON_CNTR_UNSET && test_bit(cntr_id, d->mbm_cntr_map);
>> +}
>> +
>> +static char *rdtgroup_mon_state_to_str(struct rdtgroup *rdtgrp,
>> + struct rdt_mon_domain *d, char *str)
>> +{
>> + char *tmp = str;
>> +
>> + /* Query the total and local event flags for the domain */
>> + if (resctrl_mbm_event_assigned(rdtgrp, d, QOS_L3_MBM_TOTAL_EVENT_ID))
>> + *tmp++ = 't';
>> +
>> + if (resctrl_mbm_event_assigned(rdtgrp, d, QOS_L3_MBM_LOCAL_EVENT_ID))
>> + *tmp++ = 'l';
>> +
>> + if (tmp == str)
>> + *tmp++ = '_';
>> +
>> + *tmp = '\0';
>> + return str;
>> +}
>> +
>
> Reinette
>
>
--
Thanks
Babu Moger
© 2016 - 2026 Red Hat, Inc.