kernel/events/core.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
commit a3c3c66670ce ("perf/core: Fix child_total_time_enabled accounting
bug at task exit") made changes to __perf_remove_from_context() to
coordinate its changes with perf_event_exit_event(), but the change are
unconditional, it impacts callpaths to __perf_remove_from_context()
other than from perf_event_exit_event(). One of the impact is to cgroup,
which is not properly handled and would cause kernel panic with high
probalibity during reboot on some system[1].
To confine the side effects, make the changes to
__perf_remove_from_context() conditional, restore to its previous state
except when DETACH_EXIT is set.
Closes: https://lore.kernel.org/lkml/20250601173603.3920-1-00107082@163.com/ [1]
Fixes: a3c3c66670ce ("perf/core: Fix child_total_time_enabled accounting bug at task exit")
Signed-off-by: David Wang <00107082@163.com>
---
kernel/events/core.c | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 95e703891b24..6a7e3f5c5af5 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2466,7 +2466,7 @@ __perf_remove_from_context(struct perf_event *event,
void *info)
{
struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
- enum perf_event_state state = PERF_EVENT_STATE_OFF;
+ enum perf_event_state exit_state = PERF_EVENT_STATE_EXIT;
unsigned long flags = (unsigned long)info;
ctx_time_update(cpuctx, ctx);
@@ -2475,19 +2475,20 @@ __perf_remove_from_context(struct perf_event *event,
* Ensure event_sched_out() switches to OFF, at the very least
* this avoids raising perf_pending_task() at this time.
*/
- if (flags & DETACH_EXIT)
- state = PERF_EVENT_STATE_EXIT;
if (flags & DETACH_DEAD) {
event->pending_disable = 1;
- state = PERF_EVENT_STATE_DEAD;
+ exit_state = PERF_EVENT_STATE_DEAD;
}
event_sched_out(event, ctx);
- perf_event_set_state(event, min(event->state, state));
+ if (flags & DETACH_EXIT)
+ perf_event_set_state(event, min(event->state, exit_state));
if (flags & DETACH_GROUP)
perf_group_detach(event);
if (flags & DETACH_CHILD)
perf_child_detach(event);
list_del_event(event, ctx);
+ if (flags & DETACH_DEAD)
+ event->state = PERF_EVENT_STATE_DEAD;
if (!pmu_ctx->nr_events) {
pmu_ctx->rotate_necessary = 0;
--
2.39.2commit a3c3c66670ce ("perf/core: Fix child_total_time_enabled accounting
bug at task exit") made changes to __perf_remove_from_context() to
coordinate its changes with perf_event_exit_event(), but the change are
unconditional, it impacts callpaths to __perf_remove_from_context()
other than from perf_event_exit_event(). One of the impact is to cgroup,
which is not properly handled and would cause kernel panic with high
probalibity during reboot on some system[1].
To confine the side effects, make the changes to
__perf_remove_from_context() conditional, restore to its previous state
except when DETACH_EXIT is set.
Closes: https://lore.kernel.org/lkml/20250601173603.3920-1-00107082@163.com/ [1]
Fixes: a3c3c66670ce ("perf/core: Fix child_total_time_enabled accounting bug at task exit")
Signed-off-by: David Wang <00107082@163.com>
---
Changes:
Address yeoreum.yun@arm.com's concern about missing cgroup event.
---
kernel/events/core.c | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 95e703891b24..e2c0f34b0789 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2466,7 +2466,7 @@ __perf_remove_from_context(struct perf_event *event,
void *info)
{
struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
- enum perf_event_state state = PERF_EVENT_STATE_OFF;
+ enum perf_event_state exit_state = PERF_EVENT_STATE_EXIT;
unsigned long flags = (unsigned long)info;
ctx_time_update(cpuctx, ctx);
@@ -2475,19 +2475,20 @@ __perf_remove_from_context(struct perf_event *event,
* Ensure event_sched_out() switches to OFF, at the very least
* this avoids raising perf_pending_task() at this time.
*/
- if (flags & DETACH_EXIT)
- state = PERF_EVENT_STATE_EXIT;
if (flags & DETACH_DEAD) {
event->pending_disable = 1;
- state = PERF_EVENT_STATE_DEAD;
+ exit_state = PERF_EVENT_STATE_DEAD;
}
event_sched_out(event, ctx);
- perf_event_set_state(event, min(event->state, state));
if (flags & DETACH_GROUP)
perf_group_detach(event);
if (flags & DETACH_CHILD)
perf_child_detach(event);
list_del_event(event, ctx);
+ if (flags & DETACH_EXIT)
+ perf_event_set_state(event, min(event->state, exit_state));
+ if (flags & DETACH_DEAD)
+ event->state = PERF_EVENT_STATE_DEAD;
if (!pmu_ctx->nr_events) {
pmu_ctx->rotate_necessary = 0;
--
2.39.2On Tue, Jun 03, 2025 at 04:33:04PM +0800, David Wang wrote:
> commit a3c3c66670ce ("perf/core: Fix child_total_time_enabled accounting
> bug at task exit") made changes to __perf_remove_from_context() to
> coordinate its changes with perf_event_exit_event(), but the change are
> unconditional, it impacts callpaths to __perf_remove_from_context()
> other than from perf_event_exit_event(). One of the impact is to cgroup,
> which is not properly handled and would cause kernel panic with high
> probalibity during reboot on some system[1].
Sorry, but no. This does not describe the problem adequately. I would
have to go read your [1] to figure out what is actually broken.
That is, having read the above, I'm still clueless as to what the actual
problem is.
> To confine the side effects, make the changes to
> __perf_remove_from_context() conditional, restore to its previous state
> except when DETACH_EXIT is set.
>
> Closes: https://lore.kernel.org/lkml/20250601173603.3920-1-00107082@163.com/ [1]
> Fixes: a3c3c66670ce ("perf/core: Fix child_total_time_enabled accounting bug at task exit")
> Signed-off-by: David Wang <00107082@163.com>
> ---
> Changes:
> Address yeoreum.yun@arm.com's concern about missing cgroup event.
> ---
> kernel/events/core.c | 11 ++++++-----
> 1 file changed, 6 insertions(+), 5 deletions(-)
>
> diff --git a/kernel/events/core.c b/kernel/events/core.c
> index 95e703891b24..e2c0f34b0789 100644
> --- a/kernel/events/core.c
> +++ b/kernel/events/core.c
> @@ -2466,7 +2466,7 @@ __perf_remove_from_context(struct perf_event *event,
> void *info)
> {
> struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
> - enum perf_event_state state = PERF_EVENT_STATE_OFF;
> + enum perf_event_state exit_state = PERF_EVENT_STATE_EXIT;
> unsigned long flags = (unsigned long)info;
>
> ctx_time_update(cpuctx, ctx);
> @@ -2475,19 +2475,20 @@ __perf_remove_from_context(struct perf_event *event,
> * Ensure event_sched_out() switches to OFF, at the very least
> * this avoids raising perf_pending_task() at this time.
> */
> - if (flags & DETACH_EXIT)
> - state = PERF_EVENT_STATE_EXIT;
> if (flags & DETACH_DEAD) {
> event->pending_disable = 1;
> - state = PERF_EVENT_STATE_DEAD;
> + exit_state = PERF_EVENT_STATE_DEAD;
> }
> event_sched_out(event, ctx);
> - perf_event_set_state(event, min(event->state, state));
> if (flags & DETACH_GROUP)
> perf_group_detach(event);
> if (flags & DETACH_CHILD)
> perf_child_detach(event);
> list_del_event(event, ctx);
> + if (flags & DETACH_EXIT)
> + perf_event_set_state(event, min(event->state, exit_state));
> + if (flags & DETACH_DEAD)
> + event->state = PERF_EVENT_STATE_DEAD;
Urgh, no. Trying to reverse engineer the above, the intent appears to be
to not set OFF.
This can be achieved by doing:
- enum perf_event_state state = PERF_EVENT_STATE_OFF;
+ enum perf_event_state state = event->state;
No other changes required. You also move the location of
perf_event_set_state(), but it is entirely unclear to me if that is
actually needed.
Worse, you split the means of setting state -- that is entirely uncalled
for.
At 2025-06-03 17:13:52, "Peter Zijlstra" <peterz@infradead.org> wrote:
>On Tue, Jun 03, 2025 at 04:33:04PM +0800, David Wang wrote:
>> commit a3c3c66670ce ("perf/core: Fix child_total_time_enabled accounting
>> bug at task exit") made changes to __perf_remove_from_context() to
>> coordinate its changes with perf_event_exit_event(), but the change are
>> unconditional, it impacts callpaths to __perf_remove_from_context()
>> other than from perf_event_exit_event(). One of the impact is to cgroup,
>> which is not properly handled and would cause kernel panic with high
>> probalibity during reboot on some system[1].
>
>Sorry, but no. This does not describe the problem adequately. I would
>have to go read your [1] to figure out what is actually broken.
>
>That is, having read the above, I'm still clueless as to what the actual
>problem is.
well, short story is commit a3c3c66670ce introduce a kernel panic when reboot the system
after perf_event_open with cgroup.
My understanding is commit a3c3c66670ce make changes to call path
perf_event_exit_event() --> __perf_remove_from_context(), but this changes affect other
call path as well, for example
perf_event_release_kernel() --> perf_remove_from_context()
(As yeoreum.yun@arm.com pointed out, the change in perf_remove_from_context() made
perf_event_set_state() happened before list_del_event(), resulting in perf_cgroup_event_disable()
not called.)
My suggestion here is to confine the effect of commit a3c3c66670ce only to call chain
perf_event_exit_event() --> __perf_remove_from_context()
(But this v2 version is totally wrong, should be ignored; it breaks commit a3c3c66670ce)
>
>> To confine the side effects, make the changes to
>> __perf_remove_from_context() conditional, restore to its previous state
>> except when DETACH_EXIT is set.
>>
>> Closes: https://lore.kernel.org/lkml/20250601173603.3920-1-00107082@163.com/ [1]
>> Fixes: a3c3c66670ce ("perf/core: Fix child_total_time_enabled accounting bug at task exit")
>> Signed-off-by: David Wang <00107082@163.com>
>> ---
>> Changes:
>> Address yeoreum.yun@arm.com's concern about missing cgroup event.
>> ---
>> kernel/events/core.c | 11 ++++++-----
>> 1 file changed, 6 insertions(+), 5 deletions(-)
>>
>> diff --git a/kernel/events/core.c b/kernel/events/core.c
>> index 95e703891b24..e2c0f34b0789 100644
>> --- a/kernel/events/core.c
>> +++ b/kernel/events/core.c
>> @@ -2466,7 +2466,7 @@ __perf_remove_from_context(struct perf_event *event,
>> void *info)
>> {
>> struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
>> - enum perf_event_state state = PERF_EVENT_STATE_OFF;
>> + enum perf_event_state exit_state = PERF_EVENT_STATE_EXIT;
>> unsigned long flags = (unsigned long)info;
>>
>> ctx_time_update(cpuctx, ctx);
>> @@ -2475,19 +2475,20 @@ __perf_remove_from_context(struct perf_event *event,
>> * Ensure event_sched_out() switches to OFF, at the very least
>> * this avoids raising perf_pending_task() at this time.
>> */
>> - if (flags & DETACH_EXIT)
>> - state = PERF_EVENT_STATE_EXIT;
>> if (flags & DETACH_DEAD) {
>> event->pending_disable = 1;
>> - state = PERF_EVENT_STATE_DEAD;
>> + exit_state = PERF_EVENT_STATE_DEAD;
>> }
>> event_sched_out(event, ctx);
>> - perf_event_set_state(event, min(event->state, state));
>> if (flags & DETACH_GROUP)
>> perf_group_detach(event);
>> if (flags & DETACH_CHILD)
>> perf_child_detach(event);
>> list_del_event(event, ctx);
>> + if (flags & DETACH_EXIT)
>> + perf_event_set_state(event, min(event->state, exit_state));
>> + if (flags & DETACH_DEAD)
>> + event->state = PERF_EVENT_STATE_DEAD;
>
>Urgh, no. Trying to reverse engineer the above, the intent appears to be
>to not set OFF.
>
>This can be achieved by doing:
>
>- enum perf_event_state state = PERF_EVENT_STATE_OFF;
>+ enum perf_event_state state = event->state;
>
>No other changes required. You also move the location of
>perf_event_set_state(), but it is entirely unclear to me if that is
>actually needed.
>
>Worse, you split the means of setting state -- that is entirely uncalled
>for.
Yes, that is very wired to me too..... commit a3c3c66670ce wants to use perf_event_set_state to update time,
but the original code use just event->state = ...
On Tue, Jun 03, 2025 at 06:44:58PM +0800, David Wang wrote: > (As yeoreum.yun@arm.com pointed out, the change in perf_remove_from_context() made > perf_event_set_state() happened before list_del_event(), resulting in perf_cgroup_event_disable() > not called.) Aah, d'0h. Let me see what we should do there. > My suggestion here is to confine the effect of commit a3c3c66670ce only to call chain > perf_event_exit_event() --> __perf_remove_from_context() > > > (But this v2 version is totally wrong, should be ignored; it breaks commit a3c3c66670ce) Right. Because we moved that state update earlier because perf_child_detach() wants up-to-date timestamps.
On Tue, Jun 03, 2025 at 02:50:56PM +0200, Peter Zijlstra wrote:
> On Tue, Jun 03, 2025 at 06:44:58PM +0800, David Wang wrote:
>
>
> > (As yeoreum.yun@arm.com pointed out, the change in perf_remove_from_context() made
> > perf_event_set_state() happened before list_del_event(), resulting in perf_cgroup_event_disable()
> > not called.)
>
> Aah, d'0h. Let me see what we should do there.
Does this help? This way event_sched_out() will call
perf_cgroup_event_disable().
diff --git a/kernel/events/core.c b/kernel/events/core.c
index f34c99f8ce8f..adbb0372825f 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2494,9 +2494,9 @@ __perf_remove_from_context(struct perf_event *event,
if (flags & DETACH_REVOKE)
state = PERF_EVENT_STATE_REVOKED;
if (flags & DETACH_DEAD) {
- event->pending_disable = 1;
state = PERF_EVENT_STATE_DEAD;
}
+ event->pending_disable = 1;
event_sched_out(event, ctx);
perf_event_set_state(event, min(event->state, state));
Hi Peter,
> On Tue, Jun 03, 2025 at 02:50:56PM +0200, Peter Zijlstra wrote:
> > On Tue, Jun 03, 2025 at 06:44:58PM +0800, David Wang wrote:
> >
> >
> > > (As yeoreum.yun@arm.com pointed out, the change in perf_remove_from_context() made
> > > perf_event_set_state() happened before list_del_event(), resulting in perf_cgroup_event_disable()
> > > not called.)
> >
> > Aah, d'0h. Let me see what we should do there.
>
> Does this help? This way event_sched_out() will call
> perf_cgroup_event_disable().
>
>
> diff --git a/kernel/events/core.c b/kernel/events/core.c
> index f34c99f8ce8f..adbb0372825f 100644
> --- a/kernel/events/core.c
> +++ b/kernel/events/core.c
> @@ -2494,9 +2494,9 @@ __perf_remove_from_context(struct perf_event *event,
> if (flags & DETACH_REVOKE)
> state = PERF_EVENT_STATE_REVOKED;
> if (flags & DETACH_DEAD) {
> - event->pending_disable = 1;
> state = PERF_EVENT_STATE_DEAD;
> }
> + event->pending_disable = 1;
I think it would break if event->state is "PERF_EVENT_STATE_ERROR".
TBH, there is the patch to solve this problem:
https://lore.kernel.org/all/20250602184049.4010919-1-yeoreum.yun@arm.com/
Does it have a problem?
Thanks.
--
Sincerely,
Yeoreum Yun
On Tue, Jun 03, 2025 at 02:22:37PM +0100, Yeoreum Yun wrote: > I think it would break if event->state is "PERF_EVENT_STATE_ERROR". Indeed. > TBH, there is the patch to solve this problem: > https://lore.kernel.org/all/20250602184049.4010919-1-yeoreum.yun@arm.com/ > > Does it have a problem? It does mean we can remove that pending_disable thing from DETACH_DEAD I think. Also let me edit the Changelog, the actual splat doesn't really have much useful information.
At 2025-06-03 20:54:40, "Peter Zijlstra" <peterz@infradead.org> wrote:
>On Tue, Jun 03, 2025 at 02:50:56PM +0200, Peter Zijlstra wrote:
>> On Tue, Jun 03, 2025 at 06:44:58PM +0800, David Wang wrote:
>>
>>
>> > (As yeoreum.yun@arm.com pointed out, the change in perf_remove_from_context() made
>> > perf_event_set_state() happened before list_del_event(), resulting in perf_cgroup_event_disable()
>> > not called.)
>>
>> Aah, d'0h. Let me see what we should do there.
>
>Does this help? This way event_sched_out() will call
>perf_cgroup_event_disable().
>
>
>diff --git a/kernel/events/core.c b/kernel/events/core.c
>index f34c99f8ce8f..adbb0372825f 100644
>--- a/kernel/events/core.c
>+++ b/kernel/events/core.c
>@@ -2494,9 +2494,9 @@ __perf_remove_from_context(struct perf_event *event,
> if (flags & DETACH_REVOKE)
> state = PERF_EVENT_STATE_REVOKED;
> if (flags & DETACH_DEAD) {
>- event->pending_disable = 1;
> state = PERF_EVENT_STATE_DEAD;
> }
>+ event->pending_disable = 1;
> event_sched_out(event, ctx);
> perf_event_set_state(event, min(event->state, state));
>
Ok, I will give it a try and update later.
At 2025-06-03 21:03:55, "David Wang" <00107082@163.com> wrote:
>
>At 2025-06-03 20:54:40, "Peter Zijlstra" <peterz@infradead.org> wrote:
>>On Tue, Jun 03, 2025 at 02:50:56PM +0200, Peter Zijlstra wrote:
>>> On Tue, Jun 03, 2025 at 06:44:58PM +0800, David Wang wrote:
>>>
>>>
>>> > (As yeoreum.yun@arm.com pointed out, the change in perf_remove_from_context() made
>>> > perf_event_set_state() happened before list_del_event(), resulting in perf_cgroup_event_disable()
>>> > not called.)
>>>
>>> Aah, d'0h. Let me see what we should do there.
>>
>>Does this help? This way event_sched_out() will call
>>perf_cgroup_event_disable().
>>
>>
>>diff --git a/kernel/events/core.c b/kernel/events/core.c
>>index f34c99f8ce8f..adbb0372825f 100644
>>--- a/kernel/events/core.c
>>+++ b/kernel/events/core.c
>>@@ -2494,9 +2494,9 @@ __perf_remove_from_context(struct perf_event *event,
>> if (flags & DETACH_REVOKE)
>> state = PERF_EVENT_STATE_REVOKED;
>> if (flags & DETACH_DEAD) {
>>- event->pending_disable = 1;
>> state = PERF_EVENT_STATE_DEAD;
>> }
>>+ event->pending_disable = 1;
>> event_sched_out(event, ctx);
>> perf_event_set_state(event, min(event->state, state));
>>
>
>Ok, I will give it a try and update later.
Sadly no, caught a kernel panic at the first round....
I tried to use perf to reproduce this, but no luck so far. Following is the code I used to reproduce.
(The code is silly, but valid I think....)
To reproduce, I use following steps:
Open two terminals:
1. In terminal A
mkdir /sys/fs/cgroup/mytest
echo $$ > /sys/fs/cgroup/mytest/cgroup.procs
2. In terminal B
[g++ following code if not done yet g++ -o profiler xx.cpp]
./profiler mytest
3. Do something in terminal A, usually I would run following command under kernel source tree
for i in {1..200}; do find ./ -name nottobefound > /dev/null; done
4. wait for 5~10mintes
5. In terminal B, ctrl-C stop the profiler
6. reboot
(On my system, with 6.15 at most 4 rounds of test would catch a kernel panic.)
I could not reproduce it with my KVM, maybe I need more trials.
Not sure whether anyone else could reproduce this.
---
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <sys/ioctl.h>
#include <linux/perf_event.h>
#include <asm/unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <poll.h>
#include <signal.h>
#include <fcntl.h>
#include <elf.h>
#include <string.h>
#include <vector>
#include <string>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <algorithm>
using namespace std;
#define MAXN 512
#define MAXCPU 128
#define error(msg) do { perror(msg); exit(1); } while(0)
static long perf_event_open(struct perf_event_attr *perf_event,
pid_t pid, int cpu, int group_fd, unsigned long flags) {
return syscall(__NR_perf_event_open, perf_event,
pid, cpu, group_fd, flags);
}
struct pollfd polls[MAXCPU];
// res for cleanup
static long long psize;
map<int, pair<void*, long long>> res;
static long long eventc = 0;
void int_exit(int _) {
for (auto x: res) {
auto y = x.second;
void* addr = y.first;
munmap(addr, (1+MAXN)*psize);
close(x.first);
}
res.clear();
printf("total %lld events collect\n", eventc);
exit(0);
}
int process_event(char *base, unsigned long long size, unsigned long long offset) {
struct perf_event_header* p = NULL;
offset%=size;
p = (struct perf_event_header*) (base+offset);
eventc++;
return p->size;
}
int main(int argc, char *argv[]) {
if (argc<2) { printf("Need cgroup name\n"); return 1; }
char xb[256];
snprintf(xb, sizeof(xb), "/sys/fs/cgroup/%s", argv[1]);
int cgroup_id = open(xb, O_CLOEXEC);
if (cgroup_id <= 0) error("error open cgroup dir");
int cpu_num = sysconf(_SC_NPROCESSORS_ONLN);
psize = sysconf(_SC_PAGE_SIZE); // getpagesize();
struct perf_event_attr attr;
memset(&attr, 0, sizeof(attr));
attr.type = PERF_TYPE_SOFTWARE;
attr.size = sizeof(attr);
attr.config = PERF_COUNT_SW_CPU_CLOCK;
attr.sample_freq = 9999;//777; // adjust it
attr.freq = 1;
attr.wakeup_events = 16;
attr.sample_type = PERF_SAMPLE_CALLCHAIN;
attr.sample_max_stack = 32;
attr.exclude_callchain_user = 1;
// start perf event
int i, k, fd;
void* addr;
for (i=0, k=0; i<cpu_num&&i<MAXCPU; i++) {
printf("attaching cpu %d\n", i);
fd = perf_event_open(&attr, cgroup_id, i, -1, PERF_FLAG_FD_CLOEXEC|PERF_FLAG_PID_CGROUP);
if (fd<0) error("fail to open perf event");
addr = mmap(NULL, (1+MAXN)*psize, PROT_READ, MAP_SHARED, fd, 0);
if (addr == MAP_FAILED) error("mmap failed");
res[fd] = {addr, 0};
polls[k].fd = fd;
polls[k].events = POLLIN;
polls[k].revents = 0;
k++;
}
signal(SIGINT, int_exit);
signal(SIGTERM, int_exit);
unsigned long long head;
int event_size;
struct perf_event_mmap_page *mp;
while (poll(polls, k, -1)>0) {
for (i=0; i<k; i++) {
if ((polls[i].revents&POLLIN)==0) continue;
fd = polls[i].fd;
addr = res[fd].first;
mp = (struct perf_event_mmap_page *)addr;
head = res[fd].second;
ioctl(fd, PERF_EVENT_IOC_PAUSE_OUTPUT, 1);
if (head>mp->data_head) head=mp->data_head;
head = mp->data_head-((mp->data_head-head)%mp->data_size);
while(head<mp->data_head) {
head += process_event((char*)addr+mp->data_offset, mp->data_size, head);
}
res[fd].second = mp->data_head;
ioctl(fd, PERF_EVENT_IOC_PAUSE_OUTPUT, 0);
}
}
int_exit(0);
return 0;
}
© 2016 - 2026 Red Hat, Inc.