1. Patchew
  2. EDK2
  3. View series

[edk2] [Patch] BaseTool: Add cache for the result of SkipAutogen.

BobCF posted 1 patch 5 years, 9 months ago
Failed in applying to current master (apply log)
BaseTools/Source/Python/AutoGen/AutoGen.py   | 4 ++++
BaseTools/Source/Python/Common/GlobalData.py | 1 +
2 files changed, 5 insertions(+)
[edk2] [Patch] BaseTool: Add cache for the result of SkipAutogen.
Posted by BobCF 5 years, 9 months ago 
Add a cache for the value of skip ModuleAutoGen
process flag. This cache can improve build performance.

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
---
 BaseTools/Source/Python/AutoGen/AutoGen.py   | 4 ++++
 BaseTools/Source/Python/Common/GlobalData.py | 1 +
 2 files changed, 5 insertions(+)

diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index 381dbc0a71..fc242992b7 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -4305,15 +4305,18 @@ class ModuleAutoGen(AutoGen):
 
     ## Decide whether we can skip the ModuleAutoGen process
     def CanSkipbyHash(self):
         if GlobalData.gUseHashCache:
             return not self.GenModuleHash()
+        return False
 
     ## Decide whether we can skip the ModuleAutoGen process
     #  If any source file is newer than the module than we cannot skip
     #
     def CanSkip(self):
+        if self.MetaFile in GlobalData.gSikpAutoGenCache:
+            return True
         if not os.path.exists(self.GetTimeStampPath()):
             return False
         #last creation time of the module
         DstTimeStamp = os.stat(self.GetTimeStampPath())[8]
 
@@ -4328,10 +4331,11 @@ class ModuleAutoGen(AutoGen):
                     return False
                 if source not in ModuleAutoGen.TimeDict :
                     ModuleAutoGen.TimeDict[source] = os.stat(source)[8]
                 if ModuleAutoGen.TimeDict[source] > DstTimeStamp:
                     return False
+        GlobalData.gSikpAutoGenCache.add(self.MetaFile)
         return True
 
     def GetTimeStampPath(self):
         if self._TimeStampPath is None:
             self._TimeStampPath = os.path.join(self.MakeFileDir, 'AutoGenTimeStamp')
diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py
index afb3d8f020..fac7cde708 100644
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -111,5 +111,6 @@ gBinCacheDest = None
 gBinCacheSource = None
 gPlatformHash = None
 gPackageHash = {}
 gModuleHash = {}
 gEnableGenfdsMultiThread = False
+gSikpAutoGenCache = set()
-- 
2.16.2.windows.1

_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
Re: [edk2] [Patch] BaseTool: Add cache for the result of SkipAutogen.
Posted by Gao, Liming 5 years, 9 months ago 
Bob:
  When will this cache take effective? 

Thanks
Liming
>-----Original Message-----
>From: Feng, Bob C
>Sent: Monday, July 02, 2018 9:16 AM
>To: edk2-devel@lists.01.org
>Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming <liming.gao@intel.com>
>Subject: [Patch] BaseTool: Add cache for the result of SkipAutogen.
>
>Add a cache for the value of skip ModuleAutoGen
>process flag. This cache can improve build performance.
>
>Contributed-under: TianoCore Contribution Agreement 1.1
>Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>Cc: Liming Gao <liming.gao@intel.com>
>---
> BaseTools/Source/Python/AutoGen/AutoGen.py   | 4 ++++
> BaseTools/Source/Python/Common/GlobalData.py | 1 +
> 2 files changed, 5 insertions(+)
>
>diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
>b/BaseTools/Source/Python/AutoGen/AutoGen.py
>index 381dbc0a71..fc242992b7 100644
>--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
>+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
>@@ -4305,15 +4305,18 @@ class ModuleAutoGen(AutoGen):
>
>     ## Decide whether we can skip the ModuleAutoGen process
>     def CanSkipbyHash(self):
>         if GlobalData.gUseHashCache:
>             return not self.GenModuleHash()
>+        return False
>
>     ## Decide whether we can skip the ModuleAutoGen process
>     #  If any source file is newer than the module than we cannot skip
>     #
>     def CanSkip(self):
>+        if self.MetaFile in GlobalData.gSikpAutoGenCache:
>+            return True
>         if not os.path.exists(self.GetTimeStampPath()):
>             return False
>         #last creation time of the module
>         DstTimeStamp = os.stat(self.GetTimeStampPath())[8]
>
>@@ -4328,10 +4331,11 @@ class ModuleAutoGen(AutoGen):
>                     return False
>                 if source not in ModuleAutoGen.TimeDict :
>                     ModuleAutoGen.TimeDict[source] = os.stat(source)[8]
>                 if ModuleAutoGen.TimeDict[source] > DstTimeStamp:
>                     return False
>+        GlobalData.gSikpAutoGenCache.add(self.MetaFile)
>         return True
>
>     def GetTimeStampPath(self):
>         if self._TimeStampPath is None:
>             self._TimeStampPath = os.path.join(self.MakeFileDir,
>'AutoGenTimeStamp')
>diff --git a/BaseTools/Source/Python/Common/GlobalData.py
>b/BaseTools/Source/Python/Common/GlobalData.py
>index afb3d8f020..fac7cde708 100644
>--- a/BaseTools/Source/Python/Common/GlobalData.py
>+++ b/BaseTools/Source/Python/Common/GlobalData.py
>@@ -111,5 +111,6 @@ gBinCacheDest = None
> gBinCacheSource = None
> gPlatformHash = None
> gPackageHash = {}
> gModuleHash = {}
> gEnableGenfdsMultiThread = False
>+gSikpAutoGenCache = set()
>--
>2.16.2.windows.1

_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
Re: [edk2] [Patch] BaseTool: Add cache for the result of SkipAutogen.
Posted by Feng, Bob C 5 years, 9 months ago 
Hi Liming,

Based on my testing, a module will enter the CanSkip() multiple times even after CanSkip() return True. 
In this patch, I cache the module after CanSkip() return True to save the cost of later CanSkip() evaluation.
I think the CanSkip() will always return True after it return True at first time in one build process.

Thanks,
Bob

-----Original Message-----
From: Gao, Liming 
Sent: Monday, July 02, 2018 10:53 AM
To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org
Subject: RE: [Patch] BaseTool: Add cache for the result of SkipAutogen.

Bob:
  When will this cache take effective? 

Thanks
Liming
>-----Original Message-----
>From: Feng, Bob C
>Sent: Monday, July 02, 2018 9:16 AM
>To: edk2-devel@lists.01.org
>Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming 
><liming.gao@intel.com>
>Subject: [Patch] BaseTool: Add cache for the result of SkipAutogen.
>
>Add a cache for the value of skip ModuleAutoGen process flag. This 
>cache can improve build performance.
>
>Contributed-under: TianoCore Contribution Agreement 1.1
>Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>Cc: Liming Gao <liming.gao@intel.com>
>---
> BaseTools/Source/Python/AutoGen/AutoGen.py   | 4 ++++
> BaseTools/Source/Python/Common/GlobalData.py | 1 +
> 2 files changed, 5 insertions(+)
>
>diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
>b/BaseTools/Source/Python/AutoGen/AutoGen.py
>index 381dbc0a71..fc242992b7 100644
>--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
>+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
>@@ -4305,15 +4305,18 @@ class ModuleAutoGen(AutoGen):
>
>     ## Decide whether we can skip the ModuleAutoGen process
>     def CanSkipbyHash(self):
>         if GlobalData.gUseHashCache:
>             return not self.GenModuleHash()
>+        return False
>
>     ## Decide whether we can skip the ModuleAutoGen process
>     #  If any source file is newer than the module than we cannot skip
>     #
>     def CanSkip(self):
>+        if self.MetaFile in GlobalData.gSikpAutoGenCache:
>+            return True
>         if not os.path.exists(self.GetTimeStampPath()):
>             return False
>         #last creation time of the module
>         DstTimeStamp = os.stat(self.GetTimeStampPath())[8]
>
>@@ -4328,10 +4331,11 @@ class ModuleAutoGen(AutoGen):
>                     return False
>                 if source not in ModuleAutoGen.TimeDict :
>                     ModuleAutoGen.TimeDict[source] = os.stat(source)[8]
>                 if ModuleAutoGen.TimeDict[source] > DstTimeStamp:
>                     return False
>+        GlobalData.gSikpAutoGenCache.add(self.MetaFile)
>         return True
>
>     def GetTimeStampPath(self):
>         if self._TimeStampPath is None:
>             self._TimeStampPath = os.path.join(self.MakeFileDir,
>'AutoGenTimeStamp')
>diff --git a/BaseTools/Source/Python/Common/GlobalData.py
>b/BaseTools/Source/Python/Common/GlobalData.py
>index afb3d8f020..fac7cde708 100644
>--- a/BaseTools/Source/Python/Common/GlobalData.py
>+++ b/BaseTools/Source/Python/Common/GlobalData.py
>@@ -111,5 +111,6 @@ gBinCacheDest = None  gBinCacheSource = None  
>gPlatformHash = None  gPackageHash = {}  gModuleHash = {}  
>gEnableGenfdsMultiThread = False
>+gSikpAutoGenCache = set()
>--
>2.16.2.windows.1

_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
Re: [edk2] [Patch] BaseTool: Add cache for the result of SkipAutogen.
Posted by Gao, Liming 5 years, 9 months ago 
OK. Reviewed-by: Liming Gao <liming.gao@intel.com>

>-----Original Message-----
>From: Feng, Bob C
>Sent: Monday, July 02, 2018 11:13 AM
>To: Gao, Liming <liming.gao@intel.com>; edk2-devel@lists.01.org
>Cc: Feng, Bob C <bob.c.feng@intel.com>
>Subject: RE: [Patch] BaseTool: Add cache for the result of SkipAutogen.
>
>Hi Liming,
>
>Based on my testing, a module will enter the CanSkip() multiple times even
>after CanSkip() return True.
>In this patch, I cache the module after CanSkip() return True to save the cost
>of later CanSkip() evaluation.
>I think the CanSkip() will always return True after it return True at first time in
>one build process.
>
>Thanks,
>Bob
>
>-----Original Message-----
>From: Gao, Liming
>Sent: Monday, July 02, 2018 10:53 AM
>To: Feng, Bob C <bob.c.feng@intel.com>; edk2-devel@lists.01.org
>Subject: RE: [Patch] BaseTool: Add cache for the result of SkipAutogen.
>
>Bob:
>  When will this cache take effective?
>
>Thanks
>Liming
>>-----Original Message-----
>>From: Feng, Bob C
>>Sent: Monday, July 02, 2018 9:16 AM
>>To: edk2-devel@lists.01.org
>>Cc: Feng, Bob C <bob.c.feng@intel.com>; Gao, Liming
>><liming.gao@intel.com>
>>Subject: [Patch] BaseTool: Add cache for the result of SkipAutogen.
>>
>>Add a cache for the value of skip ModuleAutoGen process flag. This
>>cache can improve build performance.
>>
>>Contributed-under: TianoCore Contribution Agreement 1.1
>>Signed-off-by: Bob Feng <bob.c.feng@intel.com>
>>Cc: Liming Gao <liming.gao@intel.com>
>>---
>> BaseTools/Source/Python/AutoGen/AutoGen.py   | 4 ++++
>> BaseTools/Source/Python/Common/GlobalData.py | 1 +
>> 2 files changed, 5 insertions(+)
>>
>>diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
>>b/BaseTools/Source/Python/AutoGen/AutoGen.py
>>index 381dbc0a71..fc242992b7 100644
>>--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
>>+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
>>@@ -4305,15 +4305,18 @@ class ModuleAutoGen(AutoGen):
>>
>>     ## Decide whether we can skip the ModuleAutoGen process
>>     def CanSkipbyHash(self):
>>         if GlobalData.gUseHashCache:
>>             return not self.GenModuleHash()
>>+        return False
>>
>>     ## Decide whether we can skip the ModuleAutoGen process
>>     #  If any source file is newer than the module than we cannot skip
>>     #
>>     def CanSkip(self):
>>+        if self.MetaFile in GlobalData.gSikpAutoGenCache:
>>+            return True
>>         if not os.path.exists(self.GetTimeStampPath()):
>>             return False
>>         #last creation time of the module
>>         DstTimeStamp = os.stat(self.GetTimeStampPath())[8]
>>
>>@@ -4328,10 +4331,11 @@ class ModuleAutoGen(AutoGen):
>>                     return False
>>                 if source not in ModuleAutoGen.TimeDict :
>>                     ModuleAutoGen.TimeDict[source] = os.stat(source)[8]
>>                 if ModuleAutoGen.TimeDict[source] > DstTimeStamp:
>>                     return False
>>+        GlobalData.gSikpAutoGenCache.add(self.MetaFile)
>>         return True
>>
>>     def GetTimeStampPath(self):
>>         if self._TimeStampPath is None:
>>             self._TimeStampPath = os.path.join(self.MakeFileDir,
>>'AutoGenTimeStamp')
>>diff --git a/BaseTools/Source/Python/Common/GlobalData.py
>>b/BaseTools/Source/Python/Common/GlobalData.py
>>index afb3d8f020..fac7cde708 100644
>>--- a/BaseTools/Source/Python/Common/GlobalData.py
>>+++ b/BaseTools/Source/Python/Common/GlobalData.py
>>@@ -111,5 +111,6 @@ gBinCacheDest = None  gBinCacheSource = None
>>gPlatformHash = None  gPackageHash = {}  gModuleHash = {}
>>gEnableGenfdsMultiThread = False
>>+gSikpAutoGenCache = set()
>>--
>>2.16.2.windows.1

_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel

Patchew 2.1-devel-0870f84

© 2016 - 2024 Red Hat, Inc.