From nobody Tue Nov 26 22:11:14 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of groups.io designates 66.175.222.12 as permitted sender) client-ip=66.175.222.12; envelope-from=bounce+27952+45172+1787277+3901457@groups.io; helo=web01.groups.io; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zoho.com: domain of groups.io designates 66.175.222.12 as permitted sender) smtp.mailfrom=bounce+27952+45172+1787277+3901457@groups.io; dmarc=fail(p=none dis=none) header.from=intel.com ARC-Seal: i=1; a=rsa-sha256; t=1565267146; cv=none; d=zoho.com; s=zohoarc; b=Ici9wry5Z2DkpIVtKvXjpD0IYksu8S9lw0NLX588lP9eWXg+8GIVXKow3pota923gBdNIQJjZuts1lvNclgP2vTCutOY41OXqR+dza5QA9S6mXP2eYCYCoBroMHnZb+sN507tj+qJAPPvhndz5f3ni2lywz3eT9Pcu6f6+gYQD0= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zoho.com; s=zohoarc; t=1565267146; h=Cc:Date:From:In-Reply-To:List-Id:List-Unsubscribe:Message-ID:Reply-To:References:Sender:Subject:To:ARC-Authentication-Results; bh=JmQ65Dw/QObXPefuZ4HGhziV8iiB2vMbQbHTSK/r4+4=; b=RjwBzbSwWbK+TTKKziJBGziHi3ip7ltnvGGRhbELHM2byvkhpe4Z7m5qoBdmjRsb0aoWkTyN0Y9B6ttpYcF8JozyxNGd14Y8b4HkBKxUzib5kfj+eaAnSh61HOhtT/TFdGLkmWWDALjy/Jy5zBwUwQipvaHWRLMce9c66WTxOvU= ARC-Authentication-Results: i=1; mx.zoho.com; dkim=pass; spf=pass (zoho.com: domain of groups.io designates 66.175.222.12 as permitted sender) smtp.mailfrom=bounce+27952+45172+1787277+3901457@groups.io; dmarc=fail header.from= (p=none dis=none) header.from= Received: from web01.groups.io (web01.groups.io [66.175.222.12]) by mx.zohomail.com with SMTPS id 1565267146940139.22933429242812; Thu, 8 Aug 2019 05:25:46 -0700 (PDT) Return-Path: X-Received: from mga14.intel.com (mga14.intel.com []) by groups.io with SMTP; Thu, 08 Aug 2019 05:25:46 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-Received: from fmsmga008.fm.intel.com ([10.253.24.58]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 08 Aug 2019 05:25:45 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,361,1559545200"; d="scan'208";a="174833155" X-Received: from jshi19-mobl.ccr.corp.intel.com ([10.254.213.128]) by fmsmga008.fm.intel.com with ESMTP; 08 Aug 2019 05:25:44 -0700 From: "Steven Shi" To: devel@edk2.groups.io Cc: liming.gao@intel.com, bob.c.feng@intel.com, christian.rodriguez@intel.com, michael.johnson@intel.com, "Shi, Steven" Subject: [edk2-devel] [PATCH 2/4] BaseTools: Print first cache missing file for build cachle Date: Thu, 8 Aug 2019 20:25:17 +0800 Message-Id: <20190808122519.25516-3-steven.shi@intel.com> In-Reply-To: <20190808122519.25516-1-steven.shi@intel.com> References: <20190808122519.25516-1-steven.shi@intel.com> Precedence: Bulk List-Unsubscribe: Sender: devel@edk2.groups.io List-Id: Mailing-List: list devel@edk2.groups.io; contact devel+owner@edk2.groups.io Reply-To: devel@edk2.groups.io,steven.shi@intel.com DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=groups.io; q=dns/txt; s=20140610; t=1565267146; bh=U6nkKv5rconvFJRpQwgHgyzd4zzanl8xeGDLM27aOIw=; h=Cc:Date:From:Reply-To:Subject:To; b=gr78PjtmM1QNEJsKBdhzLsUFRF/xB74MQFSr4zU7kfXw2ll2OySyN2yIQPyf6FijDJP o3HZeuc4i3lKcQXccDEntpfjOv0KCwSQa5wMHm+wlSrXucfYi72FMROlepAHPpCU0wJfM Dy7TEP+QOYiR/k0I2mMti+j7qADU2UQk5t0= X-ZohoMail-DKIM: pass (identity @groups.io) Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" From: "Shi, Steven" BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=3D1925 When a module build cache miss, add support to print the first cache missing file path and name. Cc: Liming Gao Cc: Bob Feng Signed-off-by: Steven Shi --- BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 2 ++ BaseTools/Source/Python/AutoGen/ModuleAutoGen.py | 76 ++++++++++++++++++++= ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/S= ource/Python/AutoGen/AutoGenWorker.py index a84ed46f2e..30d2f96fc7 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -246,6 +246,8 @@ class AutoGenWorkerInProcess(mp.Process): Ma.GenMakeHash(GlobalData.gCacheIR) if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR): continue + else: + Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR) except Empty: pass except: diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/S= ource/Python/AutoGen/ModuleAutoGen.py index 5749b8a9fa..67875f7532 100755 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -2368,6 +2368,82 @@ class ModuleAutoGen(AutoGen): print("[cache hit]: checkpoint_Makefile:", self.MetaFile.Path, sel= f.Arch) return True =20 + ## Show the first file name which causes cache miss + def PrintFirstMakeCacheMissFile(self, gDict): + if not GlobalData.gBinCacheSource: + return + + # skip binary module + if self.IsBinaryModule: + return + + if not (self.MetaFile.Path, self.Arch) in gDict: + return + + # Only print cache miss file for the MakeCache not hit module + if gDict[(self.MetaFile.Path, self.Arch)].MakeCacheHit: + return + + if not gDict[(self.MetaFile.Path, self.Arch)].MakeHashChain: + EdkLogger.quiet("[cache insight]: MakeHashChain is missing for= : %s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + # Find the cache dir name through the .ModuleHashPair file info + FileDir =3D path.join(GlobalData.gBinCacheSource, self.PlatformInf= o.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.Sourc= eDir, self.MetaFile.BaseName) + + ModuleHashPairList =3D [] # tuple list: [tuple(PreMakefileHash, Ma= keHash)] + ModuleHashPair =3D path.join(FileDir, self.Name + ".ModuleHashPair= ") + if not os.path.exists(ModuleHashPair): + EdkLogger.quiet("[cache insight]: Cannot find ModuleHashPair f= ile for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + try: + f =3D open(ModuleHashPair, 'r') + ModuleHashPairList =3D json.load(f) + f.close() + except: + EdkLogger.quiet("[cache insight]: Cannot load ModuleHashPair f= ile for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + MakeHashSet =3D set() + for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairL= ist): + TargetHashDir =3D path.join(FileDir, str(MakeHash)) + if os.path.exists(TargetHashDir): + MakeHashSet.add(MakeHash) + if not MakeHashSet: + EdkLogger.quiet("[cache insight]: Cannot find valid cache dir = for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + TargetHash =3D list(MakeHashSet)[0] + TargetHashDir =3D path.join(FileDir, str(TargetHash)) + if len(MakeHashSet) > 1 : + EdkLogger.quiet("[cache insight]: found multiple cache dirs fo= r this module, random select dir '%s' to search the first cache miss file: = %s[%s]" % (TargetHash, self.MetaFile.Path, self.Arch)) + + ListFile =3D path.join(TargetHashDir, self.Name + '.MakeHashChain') + if os.path.exists(ListFile): + try: + f =3D open(ListFile, 'r') + CachedList =3D json.load(f) + f.close() + except: + EdkLogger.quiet("[cache insight]: Cannot load MakeHashChai= n file: %s" % ListFile) + return + else: + EdkLogger.quiet("[cache insight]: Cannot find MakeHashChain fi= le: %s" % ListFile) + return + + CurrentList =3D gDict[(self.MetaFile.Path, self.Arch)].MakeHashCha= in + for idx, (file, hash) in enumerate (CurrentList): + (filecached, hashcached) =3D CachedList[idx] + if file !=3D filecached: + EdkLogger.quiet("[cache insight]: first different file in = %s[%s] is %s, the cached one is %s" % (self.MetaFile.Path, self.Arch, file,= filecached)) + break + if hash !=3D hashcached: + EdkLogger.quiet("[cache insight]: first cache miss file in= %s[%s] is %s" % (self.MetaFile.Path, self.Arch, file)) + break + + return True + ## Decide whether we can skip the ModuleAutoGen process def CanSkipbyCache(self, gDict): # Hashing feature is off --=20 2.17.1 -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#45172): https://edk2.groups.io/g/devel/message/45172 Mute This Topic: https://groups.io/mt/32797739/1787277 Group Owner: devel+owner@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [importer@patchew.org] -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-