From nobody Mon Feb 9 00:43:08 2026 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of groups.io designates 66.175.222.12 as permitted sender) client-ip=66.175.222.12; envelope-from=bounce+27952+44496+1787277+3901457@groups.io; helo=web01.groups.io; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zoho.com: domain of groups.io designates 66.175.222.12 as permitted sender) smtp.mailfrom=bounce+27952+44496+1787277+3901457@groups.io; dmarc=fail(p=none dis=none) header.from=intel.com ARC-Seal: i=1; a=rsa-sha256; t=1564389914; cv=none; d=zoho.com; s=zohoarc; b=NpMNH7lMb1wYyWdkNP3WiXftBRqAOUjqmJGZo6mGR3orEUGP6MynfrHbKCwumnihp+jX+CAyOTXaadFx4mctp4smUJpS7ZakO23jtRObz85r04qTLvgE5YpwtMpdx6gFz0hMeY1OvWkIO1ojDii0dQnxF8bSyQnwYUCJGSJD73g= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zoho.com; s=zohoarc; t=1564389914; h=Content-Transfer-Encoding:Cc:Date:From:In-Reply-To:List-Id:List-Unsubscribe:MIME-Version:Message-ID:Reply-To:References:Sender:Subject:To:ARC-Authentication-Results; bh=2aeAE/lawc5TelsY4r00xo/p5Pk5Q8+lTTUFbOFc/0A=; b=YSSsFiGXPtEIAbdxRu6XdIGH0Q1GQpEY8a50CpN6IYoNji832Ci/UJl07V4XPZBEnVMcXxp2T2qZCS//CRg1PnB0u+QOtcEviDnZ5jkFPrw9vmBuPQjiVsmUoP4aBBeYoq+8QaMgYBi5qf1ERDAESWYohbtqs82msqorQKlJHv4= ARC-Authentication-Results: i=1; mx.zoho.com; dkim=pass; spf=pass (zoho.com: domain of groups.io designates 66.175.222.12 as permitted sender) smtp.mailfrom=bounce+27952+44496+1787277+3901457@groups.io; dmarc=fail header.from= (p=none dis=none) header.from= Received: from web01.groups.io (web01.groups.io [66.175.222.12]) by mx.zohomail.com with SMTPS id 1564389914817677.0429638146543; Mon, 29 Jul 2019 01:45:14 -0700 (PDT) Return-Path: X-Received: from mga01.intel.com (mga01.intel.com []) by groups.io with SMTP; Mon, 29 Jul 2019 01:45:13 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-Received: from orsmga004.jf.intel.com ([10.7.209.38]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 29 Jul 2019 01:45:13 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,322,1559545200"; d="scan'208";a="322786663" X-Received: from shwdepsi1121.ccr.corp.intel.com ([10.239.158.47]) by orsmga004.jf.intel.com with ESMTP; 29 Jul 2019 01:45:11 -0700 From: "Bob Feng" To: devel@edk2.groups.io Cc: Liming Gao , Bob Feng Subject: [edk2-devel] [Patch 05/11] BaseTools: Enable Multiple Process AutoGen Date: Mon, 29 Jul 2019 16:44:50 +0800 Message-Id: <20190729084456.18844-6-bob.c.feng@intel.com> In-Reply-To: <20190729084456.18844-1-bob.c.feng@intel.com> References: <20190729084456.18844-1-bob.c.feng@intel.com> MIME-Version: 1.0 Precedence: Bulk List-Unsubscribe: Sender: devel@edk2.groups.io List-Id: Mailing-List: list devel@edk2.groups.io; contact devel+owner@edk2.groups.io Reply-To: devel@edk2.groups.io,bob.c.feng@intel.com Content-Transfer-Encoding: quoted-printable DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=groups.io; q=dns/txt; s=20140610; t=1564389914; bh=WUGtg3ld5ZpO2Iepamb9NXIMun3pNwFYjduOBgRWMBs=; h=Cc:Date:From:Reply-To:Subject:To; b=RnvqFjoPgu2jdDA7ca5obQFFSgDXV9woxlZCNGAsty3MK3hRiJRG2z0CSVSYgm3rhsR DwOGakC7n0DY4vDRG+xnmbLw8lpUpAT+iUI9UXFk3/RFlI8LKeUhIIeNaG7dUSg55DLjr Ie2jzn4h9A3R+fe7UFoRwRF8IPW+m8PuThs= X-ZohoMail-DKIM: pass (identity @groups.io) Content-Type: text/plain; charset="utf-8" BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=3D1875 Assign the Module AutoGen tasks into multiple sub process. Cc: Liming Gao Signed-off-by: Bob Feng --- .../Source/Python/AutoGen/AutoGenWorker.py | 162 ++++++++++++++++++ BaseTools/Source/Python/AutoGen/DataPipe.py | 6 + BaseTools/Source/Python/AutoGen/GenC.py | 4 +- .../Source/Python/AutoGen/ModuleAutoGen.py | 7 +- .../Source/Python/AutoGen/PlatformAutoGen.py | 4 +- BaseTools/Source/Python/build/build.py | 114 +++++++----- 6 files changed, 243 insertions(+), 54 deletions(-) create mode 100644 BaseTools/Source/Python/AutoGen/AutoGenWorker.py diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/S= ource/Python/AutoGen/AutoGenWorker.py new file mode 100644 index 000000000000..edec346abd06 --- /dev/null +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -0,0 +1,162 @@ +## @file +# Create makefile for MS nmake and GNU make +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from __future__ import absolute_import +import multiprocessing as mp +import threading +from Common.Misc import PathClass +from AutoGen.ModuleAutoGen import ModuleAutoGen +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo +import Common.GlobalData as GlobalData +import Common.EdkLogger as EdkLogger +import os +from Common.MultipleWorkspace import MultipleWorkspace as mws +from AutoGen.AutoGen import AutoGen +from Workspace.WorkspaceDatabase import BuildDB +import time +from queue import Empty +import traceback +import sys +from AutoGen.DataPipe import MemoryDataPipe +class AutoGenManager(threading.Thread): + def __init__(self,autogen_workers, feedback_q): + super(AutoGenManager,self).__init__() + self.autogen_workers =3D autogen_workers + self.feedback_q =3D feedback_q + self.terminate =3D False + self.Status =3D True + def run(self): + try: + while True: + if self.terminate: + break + if self.feedback_q.empty(): + time.sleep(1) + continue + badnews =3D self.feedback_q.get(False) + if badnews: + print(badnews) + self.Status =3D False + self.TerminateWorkers() + break + except Exception: + return + + def kill(self): + self.terminate =3D True + + def TerminateWorkers(self): + for w in self.autogen_workers: + if w.is_alive(): + w.terminate() + +class AutoGenWorkerInProcess(mp.Process): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_loc= k): + mp.Process.__init__(self) + self.module_queue =3D module_queue + self.data_pipe_file_path =3Ddata_pipe_file_path + self.data_pipe =3D None + self.feedback_q =3D feedback_q + self.PlatformMetaFileSet =3D {} + self.file_lock =3D file_lock + def GetPlatformMetaFile(self,filepath,root): + try: + return self.PlatformMetaFileSet[(filepath,root)] + except: + self.PlatformMetaFileSet[(filepath,root)] =3D filepath + return self.PlatformMetaFileSet[(filepath,root)] + def run(self): + try: + taskname =3D "Init" + with self.file_lock: + if not os.path.exists(self.data_pipe_file_path): + self.feedback_q.put(taskname + ":" + "load data pipe %= s failed." % self.data_pipe_file_path) + self.data_pipe =3D MemoryDataPipe() + self.data_pipe.load(self.data_pipe_file_path) + EdkLogger.Initialize() + loglevel =3D self.data_pipe.Get("LogLevel") + if not loglevel: + loglevel =3D EdkLogger.INFO + EdkLogger.SetLevel(loglevel) + logfile =3D self.data_pipe.Get("LogFile") + if logfile: + EdkLogger.SetLogFile(logfile) + target =3D self.data_pipe.Get("P_Info").get("Target") + toolchain =3D self.data_pipe.Get("P_Info").get("ToolChain") + archlist =3D self.data_pipe.Get("P_Info").get("ArchList") + + active_p =3D self.data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir =3D self.data_pipe.Get("P_Info").get("WorkspaceDi= r") + PackagesPath =3D os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + self.Wa =3D WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + self.Wa._SrcTimeStamp =3D self.data_pipe.Get("Workspace_timest= amp") + GlobalData.gGlobalDefines =3D self.data_pipe.Get("G_defines") + GlobalData.gCommandLineDefines =3D self.data_pipe.Get("CL_defi= nes") + os.environ._data =3D self.data_pipe.Get("Env_Var") + GlobalData.gWorkspace =3D workspacedir + GlobalData.gDisableIncludePathCheck =3D False + GlobalData.gFdfParser =3D self.data_pipe.Get("FdfParser") + GlobalData.gDatabasePath =3D self.data_pipe.Get("DatabasePath") + module_count =3D 0 + FfsCmd =3D self.data_pipe.Get("FfsCommand") + if FfsCmd is None: + FfsCmd =3D {} + PlatformMetaFile =3D self.GetPlatformMetaFile(self.data_pipe.G= et("P_Info").get("ActivePlatform"), + self.data_pipe.Get("P_Info").= get("WorkspaceDir")) + while not self.module_queue.empty(): + module_count +=3D 1 + module_file,module_root,module_path,module_basename,module= _originalpath,module_arch,IsLib =3D self.module_queue.get() + modulefullpath =3D os.path.join(module_root,module_file) + taskname =3D " : ".join((modulefullpath,module_arch)) + module_metafile =3D PathClass(module_file,module_root) + if module_path: + module_metafile.Path =3D module_path + if module_basename: + module_metafile.BaseName =3D module_basename + if module_originalpath: + module_metafile.OriginalPath =3D PathClass(module_orig= inalpath,module_root) + arch =3D module_arch + target =3D self.data_pipe.Get("P_Info").get("Target") + toolchain =3D self.data_pipe.Get("P_Info").get("ToolChain") + Ma =3D ModuleAutoGen(self.Wa,module_metafile,target,toolch= ain,arch,PlatformMetaFile,self.data_pipe) + Ma.IsLibrary =3D IsLib + Ma.CreateCodeFile() + Ma.CreateMakeFile(GenFfsList=3DFfsCmd.get((Ma.MetaFile.Fil= e, Ma.Arch),[])) + Ma.CreateAsBuiltInf() + except Empty: + pass + except: + traceback.print_exc(file=3Dsys.stdout) + self.feedback_q.put(taskname) + + def printStatus(self): + print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),= len(AutoGen.Cache()))) + print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpi= d(),len(AutoGenInfo.GetCache()))) + groupobj =3D {} + for buildobj in BuildDB.BuildObject.GetCache().values(): + if str(buildobj).lower().endswith("dec"): + try: + groupobj['dec'].append(str(buildobj)) + except: + groupobj['dec'] =3D [str(buildobj)] + if str(buildobj).lower().endswith("dsc"): + try: + groupobj['dsc'].append(str(buildobj)) + except: + groupobj['dsc'] =3D [str(buildobj)] + + if str(buildobj).lower().endswith("inf"): + try: + groupobj['inf'].append(str(buildobj)) + except: + groupobj['inf'] =3D [str(buildobj)] + + print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(grou= pobj.get("dec",[])))) + print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(grou= pobj.get("dsc",[])))) + print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(grou= pobj.get("inf",[])))) diff --git a/BaseTools/Source/Python/AutoGen/DataPipe.py b/BaseTools/Source= /Python/AutoGen/DataPipe.py index 5bcc39bd380d..9478f41d481b 100644 --- a/BaseTools/Source/Python/AutoGen/DataPipe.py +++ b/BaseTools/Source/Python/AutoGen/DataPipe.py @@ -9,10 +9,11 @@ from Workspace.WorkspaceDatabase import BuildDB from Workspace.WorkspaceCommon import GetModuleLibInstances import Common.GlobalData as GlobalData import os import pickle from pickle import HIGHEST_PROTOCOL +from Common import EdkLogger =20 class PCD_DATA(): def __init__(self,TokenCName,TokenSpaceGuidCName,Type,DatumType,SkuInf= oList,DefaultValue, MaxDatumSize,UserDefinedDefaultStoresFlag,validateranges, validlists,expressions,CustomAttribute,TokenValue): @@ -32,17 +33,19 @@ class PCD_DATA(): =20 class DataPipe(object): def __init__(self, BuildDir=3DNone): self.data_container =3D {} self.BuildDir =3D BuildDir + self.dump_file =3D "" =20 class MemoryDataPipe(DataPipe): =20 def Get(self,key): return self.data_container.get(key) =20 def dump(self,file_path): + self.dump_file =3D file_path with open(file_path,'wb') as fd: pickle.dump(self.data_container,fd,pickle.HIGHEST_PROTOCOL) =20 def load(self,file_path): with open(file_path,'rb') as fd: @@ -141,7 +144,10 @@ class MemoryDataPipe(DataPipe): =20 self.DataContainer =3D {"PackageList": [(dec.MetaFile,dec.Arch) fo= r dec in PlatformInfo.PackageList]} =20 self.DataContainer =3D {"GuidDict": PlatformInfo.Platform._GuidDic= t} =20 + self.DataContainer =3D {"DatabasePath":GlobalData.gDatabasePath} self.DataContainer =3D {"FdfParser": True if GlobalData.gFdfParser= else False} =20 + self.DataContainer =3D {"LogLevel": EdkLogger.GetLevel()} + self.DataContainer =3D {"LogFile": GlobalData.gOptions.LogFile if = GlobalData.gOptions.LogFile is not None else ""} diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Pyt= hon/AutoGen/GenC.py index 4c3f4e3e55ae..910c8fe3706c 100644 --- a/BaseTools/Source/Python/AutoGen/GenC.py +++ b/BaseTools/Source/Python/AutoGen/GenC.py @@ -1470,12 +1470,12 @@ def CreateModuleEntryPointCode(Info, AutoGenC, Auto= GenH): 'UefiSpecVersion': UefiSpecVersion + 'U' } =20 if Info.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_DXE_CORE, SUP_M= ODULE_SMM_CORE, SUP_MODULE_MM_CORE_STANDALONE]: if Info.SourceFileList: - if NumEntryPoints !=3D 1: - EdkLogger.error( + if NumEntryPoints !=3D 1: + EdkLogger.error( "build", AUTOGEN_ERROR, '%s must have exactly one entry point' % Info.ModuleType, File=3Dstr(Info), ExtraData=3D ", ".join(Info.Module.ModuleEntryPointList) diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/S= ource/Python/AutoGen/ModuleAutoGen.py index f0a4afc3a664..36bbaffa56d3 100644 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -1681,13 +1681,11 @@ class ModuleAutoGen(AutoGen): if self.IsBinaryModule: return =20 self.GenFfsList =3D GenFfsList =20 - if not self.IsLibrary and CreateLibraryMakeFile: - for LibraryAutoGen in self.LibraryAutoGenList: - LibraryAutoGen.CreateMakeFile() + # Don't enable if hash feature enabled, CanSkip uses timestamps to= determine build skipping if not GlobalData.gUseHashCache and self.CanSkip(): return =20 if len(self.CustomMakefile) =3D=3D 0: @@ -1724,13 +1722,10 @@ class ModuleAutoGen(AutoGen): if self.IsBinaryModule: if self.IsLibrary: self.CopyBinaryFiles() return =20 - if not self.IsLibrary and CreateLibraryCodeFile: - for LibraryAutoGen in self.LibraryAutoGenList: - LibraryAutoGen.CreateCodeFile() =20 # Don't enable if hash feature enabled, CanSkip uses timestamps to= determine build skipping if not GlobalData.gUseHashCache and self.CanSkip(): return =20 diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools= /Source/Python/AutoGen/PlatformAutoGen.py index 6360d4cbd86b..9885c6a3a3bf 100644 --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py @@ -1081,14 +1081,14 @@ class PlatformAutoGen(AutoGen): def GetAllModuleInfo(self,WithoutPcd=3DTrue): ModuleLibs =3D set() for m in self.Platform.Modules: module_obj =3D self.BuildDatabase[m,self.Arch,self.BuildTarget= ,self.ToolChain] Libs =3D GetModuleLibInstances(module_obj, self.Platform, self= .BuildDatabase, self.Arch,self.BuildTarget,self.ToolChain) - ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.Arc= h,True) for l in Libs])) + ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.Met= aFile.Path,l.MetaFile.BaseName,l.MetaFile.OriginalPath,l.Arch,True) for l i= n Libs])) if WithoutPcd and module_obj.PcdIsDriver: continue - ModuleLibs.add((m.File,m.Root,module_obj.Arch,False)) + ModuleLibs.add((m.File,m.Root,m.Path,m.BaseName,m.OriginalPath= ,module_obj.Arch,bool(module_obj.LibraryClass))) =20 return ModuleLibs =20 ## Resolve the library classes in a module to library instances # diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Pyth= on/build/build.py index 3d083f4eaade..590584b3c67f 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -28,10 +28,11 @@ from subprocess import Popen,PIPE from collections import OrderedDict, defaultdict from optparse import OptionParser from AutoGen.PlatformAutoGen import PlatformAutoGen from AutoGen.ModuleAutoGen import ModuleAutoGen from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen +from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager from AutoGen import GenMake from Common import Misc as Utils =20 from Common.TargetTxtClassObject import TargetTxt from Common.ToolDefClassObject import ToolDef @@ -48,11 +49,11 @@ from BuildReport import BuildReport from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFro= mMapFile from PatchPcdValue.PatchPcdValue import PatchBinaryFile =20 import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi - +import multiprocessing as mp =20 # Version and Copyright VersionNumber =3D "0.60" + ' ' + gBUILD_VERSION __version__ =3D "%prog Version " + VersionNumber __copyright__ =3D "Copyright (c) 2007 - 2018, Intel Corporation All right= s reserved." @@ -341,13 +342,13 @@ class ModuleMakeUnit(BuildUnit): # # @param self The object pointer # @param Obj The ModuleAutoGen object the build is working = on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency =3D [ModuleMakeUnit(La, Target) for La in Obj.LibraryAu= toGenList] - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency= , Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand,Target): + Dependency =3D [ModuleMakeUnit(La, BuildCommand,Target) for La in = Obj.LibraryAutoGenList] + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Ob= j.MakeFileDir) if Target in [None, "", "all"]: self.Target =3D "tbuild" =20 ## The smallest platform unit that can be built by nmake/make command in m= ulti-thread build mode # @@ -362,14 +363,14 @@ class PlatformMakeUnit(BuildUnit): # # @param self The object pointer # @param Obj The PlatformAutoGen object the build is workin= g on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency =3D [ModuleMakeUnit(Lib, Target) for Lib in self.BuildO= bject.LibraryAutoGenList] - Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.Bui= ldObject.ModuleAutoGenList]) - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency= , Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand, Target): + Dependency =3D [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib = in self.BuildObject.LibraryAutoGenList] + Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mo= d in self.BuildObject.ModuleAutoGenList]) + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Ob= j.MakeFileDir) =20 ## The class representing the task of a module build or platform build # # This class manages the build tasks in multi-thread build mode. Its jobs = include # scheduling thread running, catching thread error, monitor the thread sta= tus, etc. @@ -822,12 +823,35 @@ class Build(): self.TargetTxt =3D TargetTxt self.ToolDef =3D ToolDef if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBu= ildPath)): self.InitBuild() =20 + self.AutoGenMgr =3D None EdkLogger.info("") os.chdir(self.WorkspaceDir) + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList): + if SkipAutoGen: + return + feedback_q =3D mp.Queue() + file_lock =3D mp.Lock() + auto_workers =3D [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file= ,feedback_q,file_lock) for _ in range(self.ThreadNumber)] + self.AutoGenMgr =3D AutoGenManager(auto_workers,feedback_q) + self.AutoGenMgr.start() + for w in auto_workers: + w.start() + if PcdMaList is not None: + for PcdMa in PcdMaList: + PcdMa.CreateCodeFile(True) + PcdMa.CreateMakeFile(GenFfsList =3D DataPipe.Get("FfsComma= nd").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) + PcdMa.CreateAsBuiltInf() + for w in auto_workers: + w.join() + rt =3D self.AutoGenMgr.Status + self.AutoGenMgr.kill() + self.AutoGenMgr.join() + self.AutoGenMgr =3D None + return rt =20 ## Load configuration # # This method will parse target.txt and get the build configurations. # @@ -1188,30 +1212,29 @@ class Build(): # @param CreateDepModuleCodeFile Flag used to indicate creating= code # for dependent modules/Libraries # @param CreateDepModuleMakeFile Flag used to indicate creating= makefile # for dependent modules/Libraries # - def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=3DTrue, C= reateDepsMakeFile=3DTrue, BuildModule=3DFalse, FfsCommand=3D{}): + def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=3DTrue, C= reateDepsMakeFile=3DTrue, BuildModule=3DFalse, FfsCommand=3DNone, PcdMaList= =3DNone): if AutoGenObject is None: return False - + if FfsCommand is None: + FfsCommand =3D {} # skip file generation for cleanxxx targets, run and fds target if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or Target =3D=3D 'genc': - self.Progress.Start("Generating code") - AutoGenObject.CreateCodeFile(CreateDepsCodeFile) - self.Progress.Stop("done!") - if Target =3D=3D "genc": - return True + mqueue =3D mp.Queue() + for m in AutoGenObject.GetAllModuleInfo: + mqueue.put(m) =20 - if not self.SkipAutoGen or Target =3D=3D 'genmake': - self.Progress.Start("Generating makefile") - AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsComman= d) - self.Progress.Stop("done!") - if Target =3D=3D "genmake": - return True + AutoGenObject.DataPipe.DataContainer =3D {"FfsCommand":FfsComm= and} + self.Progress.Start("Generating makefile and code") + data_pipe_file =3D os.path.join(AutoGenObject.BuildDir, "Globa= lVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) + AutoGenObject.DataPipe.dump(data_pipe_file) + autogen_rt =3D self.StartAutoGen(mqueue, AutoGenObject.DataPip= e, self.SkipAutoGen, PcdMaList) + self.Progress.Stop("done!") + return autogen_rt else: # always recreate top/platform makefile when clean, just in ca= se of inconsistency AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) =20 @@ -1234,11 +1257,10 @@ class Build(): =20 # build modules if BuildModule: BuildCommand =3D BuildCommand + [Target] LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules =3D [] return True =20 @@ -1255,11 +1277,10 @@ class Build(): NewBuildCommand =3D BuildCommand + ['-f', os.path.normpath= (os.path.join(Lib, makefile)), 'pbuild'] LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) for Mod in AutoGenObject.ModuleBuildDirectoryList: NewBuildCommand =3D BuildCommand + ['-f', os.path.normpath= (os.path.join(Mod, makefile)), 'pbuild'] LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules =3D [] return True =20 @@ -1713,11 +1734,11 @@ class Build(): continue if Ma.PcdIsDriver: Ma.PlatformInfo =3D Pa PcdMaList.append(Ma) self.BuildModules.append(Ma) - self._BuildPa(self.Target, Pa, FfsCommand=3DCmdListDic= t) + self._BuildPa(self.Target, Pa, FfsCommand=3DCmdListDic= t,PcdMaList=3DPcdMaList) =20 # Create MAP file when Load Fix Address is enabled. if self.Target in ["", "all", "fds"]: for Arch in Wa.ArchList: GlobalData.gGlobalDefines['ARCH'] =3D Arch @@ -1848,11 +1869,11 @@ class Build(): GlobalData.gModuleBuildTracking[Ma.Arch][M= a] =3D 'FAIL' self.AutoGenTime +=3D int(round((time.time() - AutoGen= Start))) MakeStart =3D time.time() for Ma in self.BuildModules: if not Ma.IsBinaryModule: - Bt =3D BuildTask.New(ModuleMakeUnit(Ma, self.T= arget)) + Bt =3D BuildTask.New(ModuleMakeUnit(Ma, Pa.Bui= ldCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platf= orm ExitFlag.set() BuildTask.WaitForComplete() @@ -1978,18 +1999,19 @@ class Build(): self.LoadFixAddress =3D Wa.Platform.LoadFixAddress self.BuildReport.AddPlatformReport(Wa) Wa.CreateMakeFile(False) =20 # Add ffs build to makefile - CmdListDict =3D None + CmdListDict =3D {} if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict =3D self._GenFfsCmd(Wa.ArchList) =20 # multi-thread exit flag ExitFlag =3D threading.Event() ExitFlag.clear() self.AutoGenTime +=3D int(round((time.time() - WorkspaceAu= toGenTime))) + BuildModules =3D [] for Arch in Wa.ArchList: PcdMaList =3D [] AutoGenStart =3D time.time() GlobalData.gGlobalDefines['ARCH'] =3D Arch Pa =3D PlatformAutoGen(Wa, self.PlatformFile, BuildTar= get, ToolChain, Arch) @@ -2003,18 +2025,21 @@ class Build(): for InfName in GlobalData.gFdfParser.Profile.InfLi= st: Inf =3D PathClass(NormPath(InfName), self.Work= spaceDir, Arch) if Inf in Pa.Platform.Modules: continue ModuleList.append(Inf) + Pa.DataPipe.DataContainer =3D {"FfsCommand":CmdListDic= t} + Pa.DataPipe.DataContainer =3D {"Workspace_timestamp": = Wa._SrcTimeStamp} for Module in ModuleList: # Get ModuleAutoGen object to generate C code file= and makefile Ma =3D ModuleAutoGen(Wa, Module, BuildTarget, Tool= Chain, Arch, self.PlatformFile,Pa.DataPipe) =20 if Ma is None: continue if Ma.PcdIsDriver: Ma.PlatformInfo =3D Pa + Ma.Workspace =3D Wa PcdMaList.append(Ma) if Ma.CanSkipbyHash(): self.HashSkipModules.append(Ma) if GlobalData.gBinCacheSource: EdkLogger.quiet("cache hit: %s[%s]" % (Ma.= MetaFile.Path, Ma.Arch)) @@ -2022,38 +2047,34 @@ class Build(): else: if GlobalData.gBinCacheSource: EdkLogger.quiet("cache miss: %s[%s]" % (Ma= .MetaFile.Path, Ma.Arch)) =20 # Not to auto-gen for targets 'clean', 'cleanlib',= 'cleanall', 'run', 'fds' - if self.Target not in ['clean', 'cleanlib', 'clean= all', 'run', 'fds']: # for target which must generate AutoGen code = and makefile - if not self.SkipAutoGen or self.Target =3D=3D = 'genc': - Ma.CreateCodeFile(True) - if self.Target =3D=3D "genc": - continue =20 - if not self.SkipAutoGen or self.Target =3D=3D = 'genmake': - if CmdListDict and self.Fdf and (Module.Fi= le, Arch) in CmdListDict: - Ma.CreateMakeFile(True, CmdListDict[Mo= dule.File, Arch]) - del CmdListDict[Module.File, Arch] - else: - Ma.CreateMakeFile(True) - if self.Target =3D=3D "genmake": - continue - self.BuildModules.append(Ma) + BuildModules.append(Ma) # Initialize all modules in tracking to 'FAIL' if Ma.Arch not in GlobalData.gModuleBuildTracking: GlobalData.gModuleBuildTracking[Ma.Arch] =3D d= ict() if Ma not in GlobalData.gModuleBuildTracking[Ma.Ar= ch]: GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = =3D 'FAIL' + mqueue =3D mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + data_pipe_file =3D os.path.join(Pa.BuildDir, "GlobalVa= r_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + autogen_rt =3D self.StartAutoGen(mqueue, Pa.DataPipe, = self.SkipAutoGen, PcdMaList) self.Progress.Stop("done!") self.AutoGenTime +=3D int(round((time.time() - AutoGen= Start))) + if not autogen_rt: + return + for Arch in Wa.ArchList: MakeStart =3D time.time() - for Ma in self.BuildModules: + for Ma in BuildModules: # Generate build task for the module if not Ma.IsBinaryModule: - Bt =3D BuildTask.New(ModuleMakeUnit(Ma, self.T= arget)) + Bt =3D BuildTask.New(ModuleMakeUnit(Ma, Pa.Bui= ldCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platf= orm ExitFlag.set() BuildTask.WaitForComplete() @@ -2078,11 +2099,10 @@ class Build(): # All modules have been put in build tasks queue. Tell tas= k scheduler # to exit if all tasks are completed # ExitFlag.set() BuildTask.WaitForComplete() - self.CreateAsBuiltInf() if GlobalData.gBinCacheDest: self.UpdateBuildCache() self.BuildModules =3D [] self.MakeTime +=3D int(round((time.time() - MakeContiue))) # @@ -2496,10 +2516,16 @@ def Main(): EdkLogger.quiet("(Python %s on %s) " % (platform.python_versio= n(), sys.platform) + traceback.format_exc()) else: EdkLogger.error(X.ToolName, FORMAT_INVALID, File=3DX.FileName,= Line=3DX.LineNumber, ExtraData=3DX.Message, RaiseError=3DFalse) ReturnCode =3D FORMAT_INVALID except KeyboardInterrupt: + if MyBuild is not None: + if MyBuild.AutoGenMgr: + MyBuild.AutoGenMgr.TerminateWorkers() + MyBuild.AutoGenMgr.kill() + # for multi-thread build exits safely + MyBuild.Relinquish() ReturnCode =3D ABORT_ERROR if Option is not None and Option.debug is not None: EdkLogger.quiet("(Python %s on %s) " % (platform.python_versio= n(), sys.platform) + traceback.format_exc()) except: if MyBuild is not None: --=20 2.20.1.windows.1 -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#44496): https://edk2.groups.io/g/devel/message/44496 Mute This Topic: https://groups.io/mt/32640231/1787277 Group Owner: devel+owner@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [importer@patchew.org] -=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-=3D-