From nobody Sun Feb 8 17:36:34 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0D95CC7EE25 for ; Fri, 9 Jun 2023 16:27:09 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S242194AbjFIQ1G (ORCPT ); Fri, 9 Jun 2023 12:27:06 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:40178 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S242007AbjFIQ0z (ORCPT ); Fri, 9 Jun 2023 12:26:55 -0400 Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 291933A8C; Fri, 9 Jun 2023 09:26:49 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1686328009; x=1717864009; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=q2cuOexLZH4SnGBwsELA7mCsEQlEVUrKhgHRwoL0rJI=; b=DUQgCQFuALj0CPIvZsu3CDrRNoUWDpgXlGfXVjXCF31FG0wWJzOdQnod Pk3elIXAjSFourS2LWSqqpE27BeqczvRTX8SDk/bkrkbqunfk2vel5Tk6 LQCI/9m2ol5EsEOB3HDji+BqFZDKlw4Z4Dy1fDrxXaDe8D3j7mAWn2bfU X2k1I8mC/75X9yguHHZRYf4uy/3x0gmV3eZynKpUL1+JVpdLbRoE+xuzX 236NiWnUzDvEfFFvVBrRlMqeT1hw2YejIWfyj2l9As4cGdUXHGxNVe/EI pSE2bQziKSYI+3WsZlAa6JC1VaKcEc/lDvlce9u/184UT7i0uz2F+qqsT A==; X-IronPort-AV: E=McAfee;i="6600,9927,10736"; a="355123928" X-IronPort-AV: E=Sophos;i="6.00,229,1681196400"; d="scan'208";a="355123928" Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 09 Jun 2023 09:26:48 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10736"; a="800269728" X-IronPort-AV: E=Sophos;i="6.00,229,1681196400"; d="scan'208";a="800269728" Received: from a0cec87da3f2.jf.intel.com (HELO worker-node-1.jf.intel.com) ([10.165.55.163]) by FMSMGA003.fm.intel.com with ESMTP; 09 Jun 2023 09:26:45 -0700 From: Weilin Wang To: Peter Zijlstra , Ingo Molnar , Arnaldo Carvalho de Melo , Jiri Olsa , Namhyung Kim , Adrian Hunter , Ian Rogers , linux-perf-users@vger.kernel.org, linux-kernel@vger.kernel.org Cc: Weilin Wang , Kan Liang , Samantha Alt , Perry Taylor , Caleb Biggers Subject: [PATCH v2 3/3] perf test: Rerun failed metrics with longer workload Date: Fri, 9 Jun 2023 09:26:25 -0700 Message-Id: <20230609162625.100897-4-weilin.wang@intel.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230609162625.100897-1-weilin.wang@intel.com> References: <20230609162625.100897-1-weilin.wang@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Rerun failed metrics with longer workload to avoid false failure because sometimes metric value test fails when running in very short amount of time. Signed-off-by: Weilin Wang --- .../tests/shell/lib/perf_metric_validation.py | 129 +++++++++++------- 1 file changed, 83 insertions(+), 46 deletions(-) diff --git a/tools/perf/tests/shell/lib/perf_metric_validation.py b/tools/p= erf/tests/shell/lib/perf_metric_validation.py index e59941089350..fd39b50371d0 100644 --- a/tools/perf/tests/shell/lib/perf_metric_validation.py +++ b/tools/perf/tests/shell/lib/perf_metric_validation.py @@ -11,8 +11,9 @@ class Validator: self.rulefname =3D rulefname self.reportfname =3D reportfname self.rules =3D None - self.collectlist=3Dmetrics - self.metrics =3D set(metrics) + self.collectlist:str =3D metrics + self.metrics =3D self.__set_metrics(metrics) + self.skiplist =3D set() self.tolerance =3D t =20 self.workloads =3D [x for x in workload.split(",") if x] @@ -41,6 +42,12 @@ class Validator: self.debug =3D debug self.fullrulefname =3D fullrulefname =20 + def __set_metrics(self, metrics=3D''): + if metrics !=3D '': + return set(metrics.split(",")) + else: + return set() + def read_json(self, filename: str) -> dict: try: with open(Path(filename).resolve(), "r") as f: @@ -113,7 +120,7 @@ class Validator: All future test(s) on this metric will fail. =20 @param name: name of the metric - @returns: list with value found in self.results; list is empty whe= n not value found. + @returns: list with value found in self.results; list is empty whe= n value is not found. """ results =3D [] data =3D self.results[ridx] if ridx in self.results else self.resu= lts[0] @@ -123,7 +130,6 @@ class Validator: elif name.replace('.', '1').isdigit(): results.append(float(name)) else: - self.errlist.append("Metric '%s' is not collected or the v= alue format is incorrect"%(name)) self.ignoremetrics.add(name) return results =20 @@ -138,27 +144,32 @@ class Validator: Failure: when metric value is negative or not provided. Metrics with negative value will be added into the self.failtests[= 'PositiveValueTest'] and self.ignoremetrics. """ - negmetric =3D set() - missmetric =3D set() + negmetric =3D dict() pcnt =3D 0 tcnt =3D 0 + rerun =3D list() for name, val in self.get_results().items(): - if val is None or val =3D=3D '': - missmetric.add(name) - self.errlist.append("Metric '%s' is not collected"%(name)) - elif val < 0: - negmetric.add("{0}(=3D{1:.4f})".format(name, val)) - self.collectlist[0].append(name) + if val < 0: + negmetric[name] =3D val + rerun.append(name) else: pcnt +=3D 1 tcnt +=3D 1 + if len(rerun) > 0: + second_results =3D dict() + self.second_test(rerun, second_results) + for name, val in second_results.items(): + if name not in negmetric: continue + if val >=3D 0: + del negmetric[name] + pcnt +=3D 1 =20 self.failtests['PositiveValueTest']['Total Tests'] =3D tcnt self.failtests['PositiveValueTest']['Passed Tests'] =3D pcnt - if len(negmetric) or len(missmetric)> 0: - self.ignoremetrics.update(negmetric) - self.ignoremetrics.update(missmetric) - self.failtests['PositiveValueTest']['Failed Tests'].append({'N= egativeValue':list(negmetric), 'MissingValue':list(missmetric)}) + if len(negmetric.keys()): + self.ignoremetrics.update(negmetric.keys()) + negmessage =3D ["{0}(=3D{1:.4f})".format(name, val) for name, = val in negmetric.items()] + self.failtests['PositiveValueTest']['Failed Tests'].append({'N= egativeValue': negmessage}) =20 return =20 @@ -259,21 +270,36 @@ class Validator: metrics =3D rule['Metrics'] passcnt =3D 0 totalcnt =3D 0 - faillist =3D [] + faillist =3D list() + failures =3D dict() + rerun =3D list() for m in metrics: totalcnt +=3D 1 result =3D self.get_value(m['Name']) - if len(result) > 0 and self.check_bound(result[0], lbv, ubv, t= ): + if len(result) > 0 and self.check_bound(result[0], lbv, ubv, t= ) or m['Name'] in self.skiplist: passcnt +=3D 1 else: - faillist.append({'MetricName':m['Name'], 'CollectedValue':= result}) - self.collectlist[0].append(m['Name']) + failures[m['Name']] =3D result + rerun.append(m['Name']) + + if len(rerun) > 0: + second_results =3D dict() + self.second_test(rerun, second_results) + for name, val in second_results.items(): + if name not in failures: continue + if self.check_bound(val, lbv, ubv, t): + passcnt +=3D 1 + del failures[name] + else: + failures[name] =3D val + self.results[0][name] =3D val =20 self.totalcnt +=3D totalcnt self.passedcnt +=3D passcnt self.failtests['SingleMetricTest']['Total Tests'] +=3D totalcnt self.failtests['SingleMetricTest']['Passed Tests'] +=3D passcnt - if len(faillist) !=3D 0: + if len(failures.keys()) !=3D 0: + faillist =3D [{'MetricName':name, 'CollectedValue':val} for na= me, val in failures.items()] self.failtests['SingleMetricTest']['Failed Tests'].append({'Ru= leIndex':rule['RuleIndex'], 'Ra= ngeLower': rule['RangeLower'], 'Ra= ngeUpper': rule['RangeUpper'], @@ -316,7 +342,7 @@ class Validator: return True =20 # Start of Collector and Converter - def convert(self, data: list, idx: int): + def convert(self, data: list, metricvalues:dict): """ Convert collected metric data from the -j output to dict of {metri= c_name:value}. """ @@ -326,20 +352,29 @@ class Validator: if "metric-unit" in result and result["metric-unit"] !=3D = "(null)" and result["metric-unit"] !=3D "": name =3D result["metric-unit"].split(" ")[1] if len(r= esult["metric-unit"].split(" ")) > 1 \ else result["metric-unit"] - if idx not in self.results: self.results[idx] =3D dict= () - self.results[idx][name.lower()] =3D result["metric-val= ue"] + metricvalues[name.lower()] =3D result["metric-value"] except ValueError as error: continue return =20 - def collect_perf(self, data_file: str, workload: str): + def _run_perf(self, metric, workload: str): + tool =3D 'perf' + command =3D [tool, 'stat', '-j', '-M', f"{metric}", "-a"] + wl =3D workload.split() + command.extend(wl) + print(" ".join(command)) + cmd =3D subprocess.run(command, stderr=3Dsubprocess.PIPE, encoding= =3D'utf-8') + data =3D [x+'}' for x in cmd.stderr.split('}\n') if x] + return data + + + def collect_perf(self, workload: str): """ Collect metric data with "perf stat -M" on given workload with -a = and -j. """ self.results =3D dict() - tool =3D 'perf' print(f"Starting perf collection") - print(f"Workload: {workload}") + print(f"Long workload: {workload}") collectlist =3D dict() if self.collectlist !=3D "": collectlist[0] =3D {x for x in self.collectlist.split(",")} @@ -353,17 +388,20 @@ class Validator: collectlist[rule["RuleIndex"]] =3D [",".join(list(set(= metrics)))] =20 for idx, metrics in collectlist.items(): - if idx =3D=3D 0: wl =3D "sleep 0.5".split() - else: wl =3D workload.split() + if idx =3D=3D 0: wl =3D "true" + else: wl =3D workload for metric in metrics: - command =3D [tool, 'stat', '-j', '-M', f"{metric}", "-a"] - command.extend(wl) - print(" ".join(command)) - cmd =3D subprocess.run(command, stderr=3Dsubprocess.PIPE, = encoding=3D'utf-8') - data =3D [x+'}' for x in cmd.stderr.split('}\n') if x] - self.convert(data, idx) - self.collectlist =3D dict() - self.collectlist[0] =3D list() + data =3D self._run_perf(metric, wl) + if idx not in self.results: self.results[idx] =3D dict() + self.convert(data, self.results[idx]) + return + + def second_test(self, collectlist, second_results): + workload =3D self.workloads[self.wlidx] + for metric in collectlist: + data =3D self._run_perf(metric, workload) + self.convert(data, second_results) + # End of Collector and Converter =20 # Start of Rule Generator @@ -381,7 +419,7 @@ class Validator: if 'MetricName' not in m: print("Warning: no metric name") continue - name =3D m['MetricName'] + name =3D m['MetricName'].lower() self.metrics.add(name) if 'ScaleUnit' in m and (m['ScaleUnit'] =3D=3D '1%' or m['= ScaleUnit'] =3D=3D '100%'): self.pctgmetrics.add(name.lower()) @@ -391,14 +429,12 @@ class Validator: =20 return =20 - def remove_unsupported_rules(self, rules, skiplist: set =3D None): - for m in skiplist: - self.metrics.discard(m) + def remove_unsupported_rules(self, rules): new_rules =3D [] for rule in rules: add_rule =3D True for m in rule["Metrics"]: - if m["Name"] not in self.metrics: + if m["Name"] in self.skiplist or m["Name"] not in self.met= rics: add_rule =3D False break if add_rule: @@ -415,15 +451,15 @@ class Validator: """ data =3D self.read_json(self.rulefname) rules =3D data['RelationshipRules'] - skiplist =3D set(data['SkipList']) - self.rules =3D self.remove_unsupported_rules(rules, skiplist) + self.skiplist =3D set([name.lower() for name in data['SkipList']]) + self.rules =3D self.remove_unsupported_rules(rules) pctgrule =3D {'RuleIndex':0, 'TestType':'SingleMetricTest', 'RangeLower':'0', 'RangeUpper': '100', 'ErrorThreshold': self.tolerance, 'Description':'Metrics in percent unit have value with= in [0, 100]', - 'Metrics': [{'Name': m} for m in self.pctgmetrics]} + 'Metrics': [{'Name': m.lower()} for m in self.pctgmetr= ics]} self.rules.append(pctgrule) =20 # Re-index all rules to avoid repeated RuleIndex @@ -479,8 +515,9 @@ class Validator: self.parse_perf_metrics() self.create_rules() for i in range(0, len(self.workloads)): + self.wlidx =3D i self._init_data() - self.collect_perf(self.datafname, self.workloads[i]) + self.collect_perf(self.workloads[i]) # Run positive value test self.pos_val_test() for r in self.rules: --=20 2.39.1