From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611361; cv=none; d=zohomail.com; s=zohoarc; b=b5HsyPJmbWw2GxmxFn9dhbpXRiww3DHJatcLMDTBKsLBJ5c0dIYsxV61MePsKHbIaRSsncusx5Po4Eq+SXpmlJKFqqaogmg6H0QuJ4MFUrgv31sAVv3tUVHOPmNK56ZeubzunUDir7fHlAG4bTp3oxoHqJk79fvjstefAjB6uCo= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611361; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=DjcSpfoGpnm0mSumDdSC9lnWbPLi9k1XBXbDw6CwUFM=; b=TIMNL1sqDI3Uk1MQ/TJjNcNPmgFE9zvvDPmAIlxWOSmd7xw1+IFdcT7mXE/yqqav7GysKiqM5aLUFeO4V6bRkrpXONJCPNj5Tw3MGmSiNV35vbrorI11+7kxfyM4+o4vrrblHsAotIyu+1XaICzV6iCWIccxTxsCt1ohbYrvbUo= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595611361407626.4431945027835; Fri, 24 Jul 2020 10:22:41 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1Oo-0005eM-Al; Fri, 24 Jul 2020 17:22:22 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1Oo-0005eH-1h for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:22:22 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3ae3a48b-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:21 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1Om-00021j-GN; Fri, 24 Jul 2020 18:22:20 +0100 X-Inumbo-ID: 3ae3a48b-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 01/11] schema: Add index for quick lookup by host Date: Fri, 24 Jul 2020 18:22:06 +0100 Message-Id: <20200724172216.28204-2-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" Signed-off-by: Ian Jackson --- schema/runvars-host-index.sql | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 schema/runvars-host-index.sql diff --git a/schema/runvars-host-index.sql b/schema/runvars-host-index.sql new file mode 100644 index 00000000..fec0b960 --- /dev/null +++ b/schema/runvars-host-index.sql @@ -0,0 +1,8 @@ +-- ##OSSTEST## 009 Preparatory +-- +-- This index helps sg-report-host-history find relevant flights. + +CREATE INDEX runvars_host_idx + ON runvars (val, flight) + WHERE name =3D'host' + OR name LIKE '%_host'; --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611365; cv=none; d=zohomail.com; s=zohoarc; b=m1QB+lFEOUwoLGKQ2gyRUIFbV93PdIprHOlBcXTFXsnMTKuUCzvW9f//JuzgMqzz8eNBAc/5j0mCqb+uxAcn5yPAui8rIeRqayunJRkufvYVZbgz7TlRw6XJvLmy4wav8D2zCVjJ0YSswlxm/usai0Fi6xOL+93q4a83F+mmcMo= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611365; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=+G4ga8tN8SlGbwuv0kY21IR2ph3OC2pYy8nlcgDL2YQ=; b=IQMfvKtBTr0meUBV8wqwPFz+bphZmORyDytDwgfJGcHZh5Q5hn8ZM0YYxForUj3hx96hFeiJQM9MZk1B3wfhnPXLmQvn/KoGJypVMREq0SHYEDhO76XGEXAZEJufUJhbn+F70i07U/szoUVgQM56U0evJfFpqDUL//JpLpgs6Us= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595611365747708.7935004282103; Fri, 24 Jul 2020 10:22:45 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1Oy-0005gU-ST; Fri, 24 Jul 2020 17:22:32 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1Oy-0005eH-0Z for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:22:32 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3b648008-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:21 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1Om-00021j-Ob; Fri, 24 Jul 2020 18:22:20 +0100 X-Inumbo-ID: 3b648008-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 02/11] sg-report-host-history: Find flight limit by flight start date Date: Fri, 24 Jul 2020 18:22:07 +0100 Message-Id: <20200724172216.28204-3-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" By default we look for anything in (roughly) the last year. This query is in fact quite fast because the flights table is small. There is still the per-host limit of $limit (2000) recent runs. Signed-off-by: Ian Jackson --- sg-report-host-history | 56 ++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/sg-report-host-history b/sg-report-host-history index 54738e68..5dd875c1 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -29,6 +29,7 @@ use POSIX; use Osstest::Executive qw(:DEFAULT :colours); =20 our $limit=3D 2000; +our $timelimit=3D 86400 * (366 + 14); our $flightlimit; our $htmlout =3D "."; our $read_existing=3D1; @@ -45,6 +46,8 @@ while (@ARGV && $ARGV[0] =3D~ m/^-/) { last if m/^--?$/; if (m/^--(limit)\=3D([1-9]\d*)$/) { $$1=3D $2; + } elsif (m/^--time-limit\=3D([1-9]\d*)$/) { + $timelimit=3D $1; } elsif (m/^--flight-limit\=3D([1-9]\d*)$/) { $flightlimit=3D $1; } elsif (restrictflight_arg($_)) { @@ -108,38 +111,33 @@ sub read_existing_logs ($) { } =20 sub computeflightsrange () { - if (!$flightlimit) { - my $flagscond =3D - '('.join(' OR ', map { "f.hostflag =3D 'blessed-$_'" } @blessings).')= '; - my $nhostsq =3D db_prepare(<execute(); - my ($nhosts) =3D $nhostsq->fetchrow_array(); - print DEBUG "COUNTED $nhosts hosts\n"; - $flightlimit =3D $nhosts * $limit * 2; - } - - my $minflightsq =3D db_prepare(<execute(); + ($minflight,) =3D $minflightsq->fetchrow_array(); + } else { + my $minflightsq =3D db_prepare(<=3D ? + ORDER BY flight ASC + LIMIT 1 END - $minflightsq->execute(); - ($minflight,) =3D $minflightsq->fetchrow_array(); + my $now =3D time // die $!; + $minflightsq->execute($now - $timelimit); + ($minflight,) =3D $minflightsq->fetchrow_array(); + } $minflight //=3D 0; =20 $flightcond =3D "(flight > $minflight)"; --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611373; cv=none; d=zohomail.com; s=zohoarc; b=POLq/fvB3MopCaOS/837qy8rg1osAhVLvgWm2FoAVWJcrfCGnLp2n8LeNk7fVkf9WhJ63l3BFfQ6F696ZofaQrLSR6qQv4QL9+AR2j62am1kbxAnjf2SoefXXmvDUUwuBcmRjCyzy0TBZfUVPqL69KDW2W/a4EOUd09Exb89ZUo= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611373; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=oHFmeLHbll3mawASOTn4yEWTg95Uv1BL1ZAhmLJv4rA=; b=mZaqKpt0nEwQmCTFLPYS1p4JgZVR7lFCTX66D9v661M9NccyMXa/tu7+7WEStl5UamWrkDVbJT4j17e3lYy67qQ8yJ5PBCl78nAlKn9gCf/7cM5C1TCY/CNMgtXzToVK2MzT5OzAtm5ZIasoOkdbO5gE55wqZ3FQgLTl9yPAW/I= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595611373488223.9623490886819; Fri, 24 Jul 2020 10:22:53 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1P4-0005il-AA; Fri, 24 Jul 2020 17:22:38 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1P3-0005eH-0m for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:22:37 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3b8d8494-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:21 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1Om-00021j-W9; Fri, 24 Jul 2020 18:22:21 +0100 X-Inumbo-ID: 3b8d8494-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 03/11] sg-report-host-history: Drop per-job debug etc. Date: Fri, 24 Jul 2020 18:22:08 +0100 Message-Id: <20200724172216.28204-4-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" This printing has a significant effect on the performance of this program, at least after we optimise various other things. Signed-off-by: Ian Jackson --- sg-report-host-history | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sg-report-host-history b/sg-report-host-history index 5dd875c1..8b409fc7 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -102,9 +102,9 @@ sub read_existing_logs ($) { my $k =3D $1; s{\%([0-9a-f]{2})}{ chr hex $1 }ge; $ch->{$k} =3D $_; - print DEBUG "GOTCACHE $hostname $k\n"; +# print DEBUG "GOTCACHE $hostname $k\n"; } - print DEBUG "GOTCACHE $hostname \@ $jr->{flight} $jr->{job} $jr->{status}= ,$jr->{name}\n"; +# print DEBUG "GOTCACHE $hostname \@ $jr->{flight} $jr->{job} $jr->{status= },$jr->{name}\n"; $tcache->{$jr->{flight},$jr->{job},$jr->{status},$jr->{name}} =3D $jr; } close H; @@ -272,7 +272,7 @@ END my @rows; my $cachehits =3D 0; foreach my $jr (@$inrows) { - print DEBUG "JOB $jr->{flight}.$jr->{job} "; + #print DEBUG "JOB $jr->{flight}.$jr->{job} "; =20 my $cacherow =3D $tcache->{$jr->{flight},$jr->{job},$jr->{status},$jr->{name}}; @@ -283,11 +283,11 @@ END =20 my $endedrow =3D jobquery($endedq, $jr, 'e'); if (!$endedrow) { - print DEBUG "no-finished\n"; + #print DEBUG "no-finished\n"; next; } - print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow; - print DEBUG ".\n"; + #print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow; + #print DEBUG ".\n"; =20 push @rows, { %$jr, %$endedrow }; } @@ -329,7 +329,7 @@ END next; } =20 - print DEBUG "JR $jr->{flight}.$jr->{job}\n"; + #print DEBUG "JR $jr->{flight}.$jr->{job}\n"; my $ir =3D jobquery($infoq, $jr, 'i'); my $ar =3D jobquery($allocdq, $jr, 'a'); my $ident =3D $jr->{name}; --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611379; cv=none; d=zohomail.com; s=zohoarc; b=A02/0dBWFMXSAQRmyxl9+69SY9kLOkHeaJq7KoL3Ta7mQd6xGyWhHSkiFlJFzNyZLbL04UAVcauRpH9cCwkqxRZD8+f5CB1TNlkLHT0G/jKYuFz36yXX/l8y9jx+8u4W93SP84/SYi6ePCt+zIQh3tFaTb+3rbJXphJawJsyXAQ= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611379; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=UB5pvkiQHjYr/geS36vmMIKsIFeGAHizPcGlbwCaI3E=; b=DHgdo0XFOFDnKGd4+tqTmWEKXZ/kq3xMX4YMO3OdH5YmolX9FJvgOsVy06cod2Co3vIiwwEzMXjX4e8MQ8fhvEzlvSz7tJP7sh2BvmrFIpWXUWHYxUeIZTzOL/0WqyGPf2XqAGkgDg2f0sQT1vNsNtGopk0JNbnPMeCtIf4nGsE= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595611379875181.142808387978; Fri, 24 Jul 2020 10:22:59 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1P8-0005kQ-Ie; Fri, 24 Jul 2020 17:22:42 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1P8-0005eH-0j for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:22:42 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3bb38216-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:22 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1On-00021j-7W; Fri, 24 Jul 2020 18:22:21 +0100 X-Inumbo-ID: 3bb38216-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 04/11] Executive: Export opendb_tests Date: Fri, 24 Jul 2020 18:22:09 +0100 Message-Id: <20200724172216.28204-5-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" sg-report-host-history is going to want this in a moment Signed-off-by: Ian Jackson --- Osstest/Executive.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Osstest/Executive.pm b/Osstest/Executive.pm index 33de3708..c1095bac 100644 --- a/Osstest/Executive.pm +++ b/Osstest/Executive.pm @@ -49,7 +49,7 @@ BEGIN { task_spec_desc findtask findtask_spec @all_lock_tabl= es restrictflight_arg restrictflight_cond report_run_getinfo report_altcolour - report_altchangecolour + report_altchangecolour opendb_tests report_blessingscond report_find_push_age_info tcpconnect_queuedaemon plan_search manual_allocation_base_jobinfo --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611390; cv=none; d=zohomail.com; s=zohoarc; b=l1pxsJdTQDlgR8I6FdwEzIxaW9HJktklma89Gm/B2lmCleulVwVS6K9aUbZ1CzR2qP8gu8HOSxc8RfckRwaK2p74ctkoQ1POy/l79wDdXGjJUgzK/Y2lXyt8c97v6YcW2D9RUk1uufndyQ5tLp6VFXb0ku9FtCBibB1mN/LFvVE= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611390; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=KuchPcKW8p+EX5UbIqI3CVN2bGBAKi9Ka+PJ95HDmxo=; b=MX2n+y3KAc2N4CtssPD5niFCMic7vEv973LbuI1iQhhBDxzk/+np98ogRsaUdeCL8jpBnP8lziv2SjqiBc9m6i6rdH0C3yDUiqyYHWQHmUq5GcORQp9gZTl+KgPxoCubpEeNF8H3UPYC4WLxTcVffj/7hQCpBT4EmpIFlRQHn28= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595611390103573.2990940014656; Fri, 24 Jul 2020 10:23:10 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PD-0005n5-Sp; Fri, 24 Jul 2020 17:22:47 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PD-0005eH-0k for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:22:47 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3b8d8495-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:22 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1On-00021j-GB; Fri, 24 Jul 2020 18:22:21 +0100 X-Inumbo-ID: 3b8d8495-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 05/11] sg-report-host-history: Add a debug print after sorting jobs Date: Fri, 24 Jul 2020 18:22:10 +0100 Message-Id: <20200724172216.28204-6-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" This helps rule this sort out as a source of slowness. Signed-off-by: Ian Jackson --- sg-report-host-history | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sg-report-host-history b/sg-report-host-history index 8b409fc7..25a0c847 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -318,6 +318,8 @@ END =20 @rows =3D sort { $b->{finished} <=3D> $a->{finished} } @rows; =20 + print DEBUG "SORTED\n"; + my $alternate =3D 0; my $wrote =3D 0; my $runvarq_hits =3D 0; --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611386; cv=none; d=zohomail.com; s=zohoarc; b=laT9QbXhQ5S/+tLJVHefoKMq6qqpsrx2R/qdNiHobiFHDZ/t3xAjLAUMtPigoLv9bcohza9yXa3vNaYrXJiqQXhx5PPVCpY6/0qDzxx9v2tt8dVyNoNTIDX5ipuKt4+vRRCPypOF7kFJV6Io9/Uug/5nS3z0tGWJ/y3ykdO26S0= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611386; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=fTIG03MZVwT9CCTCuvNKYrcbgpJIzT012bdVVh6xzMo=; b=ksLGHqKqcKj0XoYO04iAapMyBo8rpjns4+plKJBMFFg8HviYJdf7zNIdxzy6qcXNDO3bhDn736SAIPCixWbmK/pTi+opnRAvLGLWoUEVXqSfkIlQ18se7sxu/QJZ723LnyrjcK3iXFbovboKXy2JdNGbV+WOfIV9j7zy4tpGmII= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595611386822206.16434942882893; Fri, 24 Jul 2020 10:23:06 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PJ-0005q3-5S; Fri, 24 Jul 2020 17:22:53 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PI-0005eH-15 for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:22:52 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3b8d8496-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:22 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1On-00021j-Ot; Fri, 24 Jul 2020 18:22:21 +0100 X-Inumbo-ID: 3b8d8496-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 06/11] sg-report-host-history: Do the main query per host Date: Fri, 24 Jul 2020 18:22:11 +0100 Message-Id: <20200724172216.28204-7-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" In f6001d628c3b3fd42b10cd15351981a04bc02572 we combined these queries into one: sg-report-host-history: Aggregate runvars query for all hosts Now that we have an index, there is a faster way for the db to do this query: via that index. But it doesn't like to do that if be aggregate the queries. Experimentally, doing this query separately once per host is significantly faster. Also, later, it will allow us to parallelise this work. So, we undo that. (Not by reverting, though.) Signed-off-by: Ian Jackson --- schema/runvars-host-index.sql | 2 +- sg-report-host-history | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/schema/runvars-host-index.sql b/schema/runvars-host-index.sql index fec0b960..cd6a1f9e 100644 --- a/schema/runvars-host-index.sql +++ b/schema/runvars-host-index.sql @@ -1,4 +1,4 @@ --- ##OSSTEST## 009 Preparatory +-- ##OSSTEST## 009 Needed -- -- This index helps sg-report-host-history find relevant flights. =20 diff --git a/sg-report-host-history b/sg-report-host-history index 25a0c847..ab88828e 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -165,34 +165,25 @@ sub jobquery ($$$) { our %hosts; =20 sub mainquery () { - our $valcond =3D join " OR ", map { "val =3D ?" } keys %hosts; - our @params =3D keys %hosts; - our $runvarq //=3D db_prepare(< ? ORDER BY flight DESC - LIMIT ($limit * 3 + 100) * ? + LIMIT $limit * 2 END + foreach my $host (sort keys %hosts) { + print DEBUG "MAINQUERY $host...\n"; + $runvarq->execute($host, $minflight); =20 - push @params, $minflight; - push @params, scalar keys %hosts; - - print DEBUG "MAINQUERY...\n"; - $runvarq->execute(@params); - - print DEBUG "FIRST PASS\n"; - while (my $jr=3D $runvarq->fetchrow_hashref()) { - print DEBUG " $jr->{flight}.$jr->{job} "; - push @{ $hosts{$jr->{val}} }, $jr; + $hosts{$host} =3D $runvarq->fetchall_arrayref({}); + print DEBUG "MAINQUERY $host got ".(scalar @{ $hosts{$host} })."\n"; } - print DEBUG "\n"; } =20 sub reporthost ($) { --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611393; cv=none; d=zohomail.com; s=zohoarc; b=Q0Sy1ZprliTSEEkxp0WJvUVaw7pdDnvlyLCl89qGpKsb5LWGOR2nIGUDWwWLRdKz9meLeRcXdCKEcZnM2HFxtuAuwJCOU9pRMN2t5gXgGosk0/jhsMmrgKRQ7o1ZXcCKSmTN6bL5RxCuM8bTpNCs6c6GQTzCpBhIfM5Zjw5kQhE= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611393; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=GGuQYGqtNKSJGc/OGWYagtrOXXtsnsU8djx9Q5vFdWw=; b=dShCLptKh3X7TpMfMY/4p0X3WkqVSln/oQ268Et2gOdLnMvYk8AGQlu7F1sq+n/Cyjv2iF8Q2rnnlcZOjSG1MCqpJMeSNDOWdaQYVTGfqsdGiir99NbPfks4AwauofHVpSnCvJOYEck3vNqLRPUnCJFkI7FY4SL4ukUjYQR293M= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 159561139364692.12876673044116; Fri, 24 Jul 2020 10:23:13 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PO-0005sY-El; Fri, 24 Jul 2020 17:22:58 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PN-0005eH-12 for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:22:57 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3c428380-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:23 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1Oo-00021j-1H; Fri, 24 Jul 2020 18:22:22 +0100 X-Inumbo-ID: 3c428380-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 07/11] sg-report-host-history: Rerganisation: Make mainquery per-host Date: Fri, 24 Jul 2020 18:22:12 +0100 Message-Id: <20200724172216.28204-8-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" This moves the loop over hosts into the main program. We are working our way to a new code structure. No functional change. Signed-off-by: Ian Jackson --- sg-report-host-history | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sg-report-host-history b/sg-report-host-history index ab88828e..509d053d 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -164,7 +164,9 @@ sub jobquery ($$$) { =20 our %hosts; =20 -sub mainquery () { +sub mainquery ($) { + my ($host) =3D @_; + our $runvarq //=3D db_prepare(<execute($host, $minflight); =20 - $hosts{$host} =3D $runvarq->fetchall_arrayref({}); - print DEBUG "MAINQUERY $host got ".(scalar @{ $hosts{$host} })."\n"; - } + print DEBUG "MAINQUERY $host...\n"; + $runvarq->execute($host, $minflight); + + $hosts{$host} =3D $runvarq->fetchall_arrayref({}); + print DEBUG "MAINQUERY $host got ".(scalar @{ $hosts{$host} })."\n"; } =20 sub reporthost ($) { @@ -473,7 +474,9 @@ db_retry($dbh_tests, [], sub { }); =20 db_retry($dbh_tests, [], sub { - mainquery(); + foreach my $host (sort keys %hosts) { + mainquery($host); + } }); =20 foreach my $host (sort keys %hosts) { --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611395; cv=none; d=zohomail.com; s=zohoarc; b=Q7L6A2mUTSiZ/6sE8z8mpvl5BQVtvWYnYFtzMMvzxSN6On4NfaFo9eTAOGsCBELjIYuZbcuw1K3rMc091XUUpmkrADSkveMH4w7ge9UHn0VDfri8UrdF8vdHmlM3U5KtTv/Q0JgeP6a/bg6QsukziZpstUjIrEANrrDwS794dwE= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611395; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=yLlhkG+m9rz+oC+8o8lj1AC4sSyEti9WNA/Pboy0Lfw=; b=LrmRuAOqeaNp7aYrv4LHuKDPVGykF6hPsAOEqoI5AClNEPNOR8Vkq4PWX+1r6gKQ429/6cUW9KdJ0Km/ppgEZ6QbRjW4A48KjCMwNvz+8rj+9yOIcIxQLxM1R90/rCcIK6DJXvDmyXrAmRi54JR3bQjl1SR20CJ8kc9C/HxttnM= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595611395216257.02900773540114; Fri, 24 Jul 2020 10:23:15 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PS-0005x0-O8; Fri, 24 Jul 2020 17:23:02 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PS-0005eH-1B for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:23:02 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3b648009-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:23 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1Oo-00021j-Dx; Fri, 24 Jul 2020 18:22:22 +0100 X-Inumbo-ID: 3b648009-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 08/11] sg-report-host-history: Rerganisation: Read old logs later Date: Fri, 24 Jul 2020 18:22:13 +0100 Message-Id: <20200724172216.28204-9-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" Perhaps at one point something read from these logs influenced the db query for thye flights range, but that is no longer the case and it doesn't seem likely to need to come back. We want to move the per-host stuff together. No functional change. Signed-off-by: Ian Jackson --- sg-report-host-history | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sg-report-host-history b/sg-report-host-history index 509d053d..4b0b5b2d 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -465,14 +465,14 @@ END =20 exit 0 unless %hosts; =20 -foreach (keys %hosts) { - read_existing_logs($_); -} - db_retry($dbh_tests, [], sub { computeflightsrange(); }); =20 +foreach (keys %hosts) { + read_existing_logs($_); +} + db_retry($dbh_tests, [], sub { foreach my $host (sort keys %hosts) { mainquery($host); --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595611402; cv=none; d=zohomail.com; s=zohoarc; b=GY9uWy3h5+wX4VXj2mLhLeKG+Y8g1ecGDsxVE88qaWfDhJTwVBNxcTSgv4EWbHwHhQ3tAuyS0kqb6mF0J4nKmuGs4rUM5hWWwy1qHePWpOjngQwrh2UVTbA71fbKt/mmWyjiKZCCCW1NQ13/kScfmrFmyAogIrNUw2/vpswRxLA= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595611402; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=QMiUrUdl3iub69OGqXXBTqG/9wTJc/FwBJNgrvFjxKE=; b=U6cMl2SEy7r5qNKNAxjlXzqAxjZe0Em1CghD0cWD0pNR3r75JLoUqXN19V9c3zcbhYKKz4hG9/5pjEoS99bnATpZuKOA846lbZ5VCWxJdPSGA0TJOXrblrObHKLgf2dV/bAp0xTB1+LeFv979u4rA0p5T+t7ljZb9SRUBlpWFDY= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595611402517994.0099784303602; Fri, 24 Jul 2020 10:23:22 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PY-00060l-2E; Fri, 24 Jul 2020 17:23:08 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1PX-0005eH-1D for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:23:07 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id 3c9c94d8-cdd2-11ea-8862-bc764e2007e4; Fri, 24 Jul 2020 17:22:23 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1Oo-00021j-N9; Fri, 24 Jul 2020 18:22:22 +0100 X-Inumbo-ID: 3c9c94d8-cdd2-11ea-8862-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 09/11] sg-report-host-history: Rerganisation: Change loops Date: Fri, 24 Jul 2020 18:22:14 +0100 Message-Id: <20200724172216.28204-10-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" Move the per-host code all into the same per-host loop. One effect is to transpose the db_retry and host loops for mainquery. No functional change. Signed-off-by: Ian Jackson --- sg-report-host-history | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/sg-report-host-history b/sg-report-host-history index 4b0b5b2d..1f5c14e1 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -469,18 +469,10 @@ db_retry($dbh_tests, [], sub { computeflightsrange(); }); =20 -foreach (keys %hosts) { - read_existing_logs($_); -} - -db_retry($dbh_tests, [], sub { - foreach my $host (sort keys %hosts) { - mainquery($host); - } -}); - foreach my $host (sort keys %hosts) { + read_existing_logs($host); db_retry($dbh_tests, [], sub { + mainquery($host); reporthost $host; }); } --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595612425; cv=none; d=zohomail.com; s=zohoarc; b=f3qOeUyLQl2jv9jrfdiuRjHfICwA98GBzjl7Vv95fO3ianc5NeB4lhwuUMyZcnwTrb20J7jEEkVKcYzgcYNDSy8w/xWLize94Wrq6qd9k66amVz5QqZXxuwXBm9Yk+cLTruwcrXiXH25wFlOmjIvJKXt7lO/5aWYtP5Cge7skUc= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595612425; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=zTIQ6p2iUh9qwTpsxLPGX29csVfcA6TXs+Sr/pTbm40=; b=UkzjAGDCH6MCfmXdx8YYhkW+jrj3FE9hJsft5lIK/keE29Xigd60aPJiY4f5w5NyDzT1O4umtkOTjSW5bVwAu1BI0hJ3DyUej9XwRp6YdsFF6i+IljL64D3sYeLbVX20pNPg9FXL6w99DgCJGumD2YDg+6a1TxtP7O6SoaZBPEQ= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595612425866223.27758671041272; Fri, 24 Jul 2020 10:40:25 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1fk-0007nl-77; Fri, 24 Jul 2020 17:39:52 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1fj-0007ng-D9 for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:39:51 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id ac5af1e6-cdd4-11ea-8867-bc764e2007e4; Fri, 24 Jul 2020 17:39:50 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1Op-00021j-0g; Fri, 24 Jul 2020 18:22:23 +0100 X-Inumbo-ID: ac5af1e6-cdd4-11ea-8867-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 10/11] sg-report-host-history: Drop a redundznt AND clause Date: Fri, 24 Jul 2020 18:22:15 +0100 Message-Id: <20200724172216.28204-11-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" This condition is the same as $flightcond. (This has no effect on the db performance since the query planner figures it out, but it is confusing.) Signed-off-by: Ian Jackson --- sg-report-host-history | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sg-report-host-history b/sg-report-host-history index 1f5c14e1..787f7c5b 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -175,13 +175,12 @@ sub mainquery ($) { AND val =3D ? AND $flightcond AND $restrictflight_cond - AND flight > ? ORDER BY flight DESC LIMIT $limit * 2 END =20 print DEBUG "MAINQUERY $host...\n"; - $runvarq->execute($host, $minflight); + $runvarq->execute($host); =20 $hosts{$host} =3D $runvarq->fetchall_arrayref({}); print DEBUG "MAINQUERY $host got ".(scalar @{ $hosts{$host} })."\n"; --=20 2.20.1 From nobody Sat Apr 20 11:31:46 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail(p=none dis=none) header.from=eu.citrix.com ARC-Seal: i=1; a=rsa-sha256; t=1595612426; cv=none; d=zohomail.com; s=zohoarc; b=ZS5Fuxwk1LoUgGK/rV28CdGlfktkR/+iLoL7tOQB8XyHOKpclzf3tzGi5Q1S+Ka9jzHY9dymjrcX8MIjqus6gAlvfxGW5htP0yuCgR7614zb4j7f2OZzxpUyvNJ4OxNVC9w0Xa5jfOsL/r2uD+u5pS/iqrDoBk0fvfn04sYOio0= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1595612426; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=ttTYQVV28fqNk18XUjObQj3rNPda8bBr2imx4x22q90=; b=gdjZvq+oF7aFW/5QIM9LpGoycenrlFuchNftbfnq/zB33Og8KU7GxpBe4uMNnhrx8N9yzVf7w8d92SRfB10cbMLSYDVIWPFk2w4tOupxYM1FDVcEXfJ4fXb35zIjg5gPDM5JZtsS1QbAncwd8FDh/QBXvWFSr/SbJNabaB6qw0w= ARC-Authentication-Results: i=1; mx.zohomail.com; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=fail header.from= (p=none dis=none) header.from= Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1595612425969932.9469971153502; Fri, 24 Jul 2020 10:40:25 -0700 (PDT) Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1fz-0008S3-Fq; Fri, 24 Jul 2020 17:40:07 +0000 Received: from us1-rack-iad1.inumbo.com ([172.99.69.81]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1jz1fy-0008OM-Pa for xen-devel@lists.xenproject.org; Fri, 24 Jul 2020 17:40:06 +0000 Received: from chiark.greenend.org.uk (unknown [2001:ba8:1e3::]) by us1-rack-iad1.inumbo.com (Halon) with ESMTPS id b5d650e5-cdd4-11ea-886a-bc764e2007e4; Fri, 24 Jul 2020 17:40:06 +0000 (UTC) Received: from [172.18.45.5] (helo=zealot.relativity.greenend.org.uk) by chiark.greenend.org.uk (Debian Exim 4.84_2 #1) with esmtp (return-path ijackson@chiark.greenend.org.uk) id 1jz1Op-00021j-A9; Fri, 24 Jul 2020 18:22:23 +0100 X-Inumbo-ID: b5d650e5-cdd4-11ea-886a-bc764e2007e4 From: Ian Jackson To: xen-devel@lists.xenproject.org Subject: [OSSTEST PATCH 11/11] sg-report-host-history: Fork Date: Fri, 24 Jul 2020 18:22:16 +0100 Message-Id: <20200724172216.28204-12-ian.jackson@eu.citrix.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200724172216.28204-1-ian.jackson@eu.citrix.com> References: <20200724172216.28204-1-ian.jackson@eu.citrix.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-BeenThere: xen-devel@lists.xenproject.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Cc: Ian Jackson Errors-To: xen-devel-bounces@lists.xenproject.org Sender: "Xen-devel" Content-Type: text/plain; charset="utf-8" Run each host's report in a separate child. This is considerably faster. Signed-off-by: Ian Jackson --- sg-report-host-history | 47 +++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/sg-report-host-history b/sg-report-host-history index 787f7c5b..d8e19127 100755 --- a/sg-report-host-history +++ b/sg-report-host-history @@ -34,6 +34,7 @@ our $flightlimit; our $htmlout =3D "."; our $read_existing=3D1; our $doinstall=3D1; +our $maxjobs=3D10; our @blessings; =20 open DEBUG, ">/dev/null"; @@ -44,7 +45,7 @@ csreadconfig(); while (@ARGV && $ARGV[0] =3D~ m/^-/) { $_=3D shift @ARGV; last if m/^--?$/; - if (m/^--(limit)\=3D([1-9]\d*)$/) { + if (m/^--(limit|maxjobs)\=3D([1-9]\d*)$/) { $$1=3D $2; } elsif (m/^--time-limit\=3D([1-9]\d*)$/) { $timelimit=3D $1; @@ -468,12 +469,44 @@ db_retry($dbh_tests, [], sub { computeflightsrange(); }); =20 +undef $dbh_tests; + +our %children; +our $worst =3D 0; + +sub wait_for_max_children ($) { + my ($lim) =3D @_; + while (keys(%children) > $lim) { + $!=3D0; $?=3D0; my $got =3D wait; + die "$! $got $?" unless exists $children{$got}; + my $host =3D $children{$got}; + delete $children{$got}; + $worst =3D $? if $? > $worst; + if ($?) { + print STDERR "sg-report-flight[: [$got] failed for $host: $?\n"; + } else { + print DEBUG "REAPED [$got] $host\n"; + } + } +} + foreach my $host (sort keys %hosts) { - read_existing_logs($host); - db_retry($dbh_tests, [], sub { - mainquery($host); - reporthost $host; - }); + wait_for_max_children($maxjobs); + + my $pid =3D fork // die $!; + if (!$pid) { + opendb_tests(); + read_existing_logs($host); + db_retry($dbh_tests, [], sub { + mainquery($host); + reporthost $host; + }); + print DEBUG "JQ CACHE ".($jqtotal-$jqcachemisses)." / $jqtotal\n"; + exit(0); + } + print DEBUG "SPAWNED [$pid] $host\n"; + $children{$pid} =3D $host; } =20 -print DEBUG "JQ CACHE ".($jqtotal-$jqcachemisses)." / $jqtotal\n"; +wait_for_max_children(0); +exit $worst; --=20 2.20.1