[edk2] [PATCH] BaseTools: Fix flexible PCD single quote and double quote bugs

Feng, YunhuaX posted 1 patch 6 years, 1 month ago
Patches applied successfully (tree, apply log)
git fetch https://github.com/patchew-project/edk2 tags/patchew/47C64442C08CCD4089DC43B6B5E46BC482B48C@shsmsx102.ccr.corp.intel.com
There is a newer version of this series
BaseTools/Source/Python/AutoGen/GenMake.py        | 20 +++++---
BaseTools/Source/Python/Common/Expression.py      | 60 ++++++++++++++++-------
BaseTools/Source/Python/Common/Misc.py            | 31 +++++++-----
BaseTools/Source/Python/Common/String.py          | 33 ++++++++-----
BaseTools/Source/Python/Workspace/DscBuildData.py | 37 +++++++-------
5 files changed, 115 insertions(+), 66 deletions(-)
[edk2] [PATCH] BaseTools: Fix flexible PCD single quote and double quote bugs
Posted by Feng, YunhuaX 6 years, 1 month ago
1. Argument --pcd format as below:
  Some examples that to match --pcd format and DSC format
  --pcd                             DSC format
  L"ABC"                            L"ABC"
  "AB\\\"C"                         "AB\"C"
  "AB\tC"                          "AB\tC"
  "AB\\\'C"                         "AB\'C"
  "\'ABC\'"                         'ABC’
  L"\'AB\\\"C\'"                    L'AB\"C'
  "\'AB\\\'C\'"                     'AB\'C'
  "\'AB\tC\'"                      'AB\tC'
  H"{0, L\"AB\\\"B\", \'ab\\\"c\'}" {0, L"AB\"B", 'ab\"c'}
  H"{0, L\"AB\\\"B\",\'ab\\\'c\'}"  {0, L"AB\"B", 'ab\'c'}
2. {'#', '|'} split string incorrectly.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1

Signed-off-by: Yunhua Feng <yunhuax.feng@intel.com>
---
 BaseTools/Source/Python/AutoGen/GenMake.py        | 20 +++++---
 BaseTools/Source/Python/Common/Expression.py      | 60 ++++++++++++++++-------
 BaseTools/Source/Python/Common/Misc.py            | 31 +++++++-----
 BaseTools/Source/Python/Common/String.py          | 33 ++++++++-----
 BaseTools/Source/Python/Workspace/DscBuildData.py | 37 +++++++-------
 5 files changed, 115 insertions(+), 66 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index afe6f2f99c..4b924d21e0 100644
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1553,17 +1553,23 @@ class TopLevelMakefile(BuildFile):
 
         if GlobalData.BuildOptionPcd:
             for index, option in enumerate(GlobalData.gCommand):
                 if "--pcd" == option and GlobalData.gCommand[index+1]:
                     pcdName, pcdValue = GlobalData.gCommand[index+1].split('=')
-                    if pcdValue.startswith('H'):
-                        pcdValue = 'H' + '"' + pcdValue[1:] + '"'
-                        ExtraOption += " --pcd " + pcdName + '=' + pcdValue
-                    elif pcdValue.startswith("L'"):
-                        ExtraOption += "--pcd " + pcdName + '=' + pcdValue
-                    elif pcdValue.startswith('L'):
-                        pcdValue = 'L' + '"' + pcdValue[1:] + '"'
+                    for Item in GlobalData.BuildOptionPcd:
+                        if '.'.join(Item[0:2]) == pcdName:
+                            pcdValue = Item[2]
+                            if pcdValue.startswith('L') or pcdValue.startswith('"'):
+                                pcdValue, Size = ParseFieldValue(pcdValue)
+                                NewVal = '{'
+                                for S in range(Size):
+                                    NewVal = NewVal + '0x%02X' % ((pcdValue >> S * 8) & 0xff)
+                                    NewVal += ','
+                                pcdValue =  NewVal[:-1] + '}'
+                            break
+                    if pcdValue.startswith('{'):
+                        pcdValue = 'H' + '"' + pcdValue + '"'
                         ExtraOption += " --pcd " + pcdName + '=' + pcdValue
                     else:
                         ExtraOption += " --pcd " + GlobalData.gCommand[index+1]
 
         MakefileName = self._FILE_NAME_[self._FileType]
diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py
index edb0a60de6..f1516d5c7b 100644
--- a/BaseTools/Source/Python/Common/Expression.py
+++ b/BaseTools/Source/Python/Common/Expression.py
@@ -43,28 +43,41 @@ ERR_IN_OPERAND          = 'Macro after IN operator can only be: $(FAMILY), $(ARC
 ## SplitString
 #  Split string to list according double quote
 #  For example: abc"de\"f"ghi"jkl"mn will be: ['abc', '"de\"f"', 'ghi', '"jkl"', 'mn']
 #
 def SplitString(String):
-    # There might be escaped quote: "abc\"def\\\"ghi"
-    Str = String.replace('\\\\', '//').replace('\\\"', '\\\'')
+    # There might be escaped quote: "abc\"def\\\"ghi", 'abc\'def\\\'ghi'
+    Str = String
     RetList = []
-    InQuote = False
+    InSingleQuote = False
+    InDoubleQuote = False
     Item = ''
     for i, ch in enumerate(Str):
-        if ch == '"':
-            InQuote = not InQuote
-            if not InQuote:
+        if ch == '"' and not InSingleQuote:
+            if Str[i - 1] != '\\':
+                InDoubleQuote = not InDoubleQuote
+            if not InDoubleQuote:
+                Item += String[i]
+                RetList.append(Item)
+                Item = ''
+                continue
+            if Item:
+                RetList.append(Item)
+                Item = ''
+        elif ch == "'" and not InDoubleQuote:
+            if Str[i - 1] != '\\':
+                InSingleQuote = not InSingleQuote
+            if not InSingleQuote:
                 Item += String[i]
                 RetList.append(Item)
                 Item = ''
                 continue
             if Item:
                 RetList.append(Item)
                 Item = ''
         Item += String[i]
-    if InQuote:
+    if InSingleQuote or InDoubleQuote:
         raise BadExpression(ERR_STRING_TOKEN % Item)
     if Item:
         RetList.append(Item)
     return RetList
 
@@ -481,17 +494,21 @@ class ValueExpression(object):
             Radix = 16
         if self._Token.startswith('"') or self._Token.startswith('L"'):
             Flag = 0
             for Index in range(len(self._Token)):
                 if self._Token[Index] in ['"']:
+                    if self._Token[Index - 1] == '\\':
+                        continue
                     Flag += 1
             if Flag == 2 and self._Token.endswith('"'):
                 return True
         if self._Token.startswith("'") or self._Token.startswith("L'"):
             Flag = 0
             for Index in range(len(self._Token)):
                 if self._Token[Index] in ["'"]:
+                    if self._Token[Index - 1] == '\\':
+                        continue
                     Flag += 1
             if Flag == 2 and self._Token.endswith("'"):
                 return True
         try:
             self._Token = int(self._Token, Radix)
@@ -535,20 +552,29 @@ class ValueExpression(object):
 
         # Skip left quote
         self._Idx += 1
 
         # Replace escape \\\", \"
-        Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'')
-        for Ch in Expr:
-            self._Idx += 1
-            if Ch == '"' or Ch == "'":
-                break
-        self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]
-        if self._Token.startswith('"') and not self._Token.endswith('"'):
-            raise BadExpression(ERR_STRING_TOKEN % self._Token)
-        if self._Token.startswith("'") and not self._Token.endswith("'"):
-            raise BadExpression(ERR_STRING_TOKEN % self._Token)
+        if self._Expr[Idx] == '"':
+            Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'')
+            for Ch in Expr:
+                self._Idx += 1
+                if Ch == '"':
+                    break
+            self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]
+            if not self._Token.endswith('"'):
+                raise BadExpression(ERR_STRING_TOKEN % self._Token)
+        #Replace escape \\\', \'
+        elif self._Expr[Idx] == "'":
+            Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace("\\\'", "\\\"")
+            for Ch in Expr:
+                self._Idx += 1
+                if Ch == "'":
+                    break
+            self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]
+            if not self._Token.endswith("'"):
+                raise BadExpression(ERR_STRING_TOKEN % self._Token)
         self._Token = self._Token[1:-1]
         return self._Token
 
     # Get token that is comprised by alphanumeric, underscore or dot(used by PCD)
     # @param IsAlphaOp: Indicate if parsing general token or script operator(EQ, NE...)
diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 1461d00669..a7e7797d04 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -1441,25 +1441,30 @@ def ParseConsoleLog(Filename):
     Opr.close()
     Opw.close()
 
 def AnalyzePcdExpression(Setting):
     Setting = Setting.strip()
-    # There might be escaped quote in a string: \", \\\"
-    Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')
+    # There might be escaped quote in a string: \", \\\" , \', \\\'
+    Data = Setting
     # There might be '|' in string and in ( ... | ... ), replace it with '-'
     NewStr = ''
-    InStr = False
+    InSingleQuoteStr = False
+    InDoubleQuoteStr = False
     Pair = 0
-    for ch in Data:
-        if ch == '"':
-            InStr = not InStr
-        elif ch == '(' and not InStr:
+    for Index, ch in enumerate(Data):
+        if ch == '"' and not InSingleQuoteStr:
+            if Data[Index - 1] != '\\':
+                InDoubleQuoteStr = not InDoubleQuoteStr
+        elif ch == "'" and not InDoubleQuoteStr:
+            if Data[Index - 1] != '\\':
+                InSingleQuoteStr = not InSingleQuoteStr
+        elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):
             Pair += 1
-        elif ch == ')' and not InStr:
+        elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):
             Pair -= 1
 
-        if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:
+        if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:
             NewStr += '-'
         else:
             NewStr += ch
     FieldList = []
     StartPos = 0
@@ -1547,37 +1552,37 @@ def ParseFieldValue (Value):
             raise BadExpression('%s' % Message)
         Value, Size = ParseFieldValue(Value)
         return Value, 16
     if Value.startswith('L"') and Value.endswith('"'):
         # Unicode String
-        List = list(Value[2:-1])
+        List = list(eval(Value[1:]))  # translate escape character
         List.reverse()
         Value = 0
         for Char in List:
             Value = (Value << 16) | ord(Char)
         return Value, (len(List) + 1) * 2
     if Value.startswith('"') and Value.endswith('"'):
         # ASCII String
-        List = list(Value[1:-1])
+        List = list(eval(Value))  # translate escape character
         List.reverse()
         Value = 0
         for Char in List:
             Value = (Value << 8) | ord(Char)
         return Value, len(List) + 1
     if Value.startswith("L'") and Value.endswith("'"):
         # Unicode Character Constant
-        List = list(Value[2:-1])
+        List = list(eval(Value[1:])) # translate escape character
         if len(List) == 0:
             raise BadExpression('Length %s is %s' % (Value, len(List)))
         List.reverse()
         Value = 0
         for Char in List:
             Value = (Value << 16) | ord(Char)
         return Value, len(List) * 2
     if Value.startswith("'") and Value.endswith("'"):
         # Character constant
-        List = list(Value[1:-1])
+        List = list(eval(Value)) # translate escape character
         if len(List) == 0:
             raise BadExpression('Length %s is %s' % (Value, len(List)))
         List.reverse()
         Value = 0
         for Char in List:
diff --git a/BaseTools/Source/Python/Common/String.py b/BaseTools/Source/Python/Common/String.py
index 4a8c03e88e..bf36783f8f 100644
--- a/BaseTools/Source/Python/Common/String.py
+++ b/BaseTools/Source/Python/Common/String.py
@@ -43,30 +43,36 @@ gHumanReadableVerPatt = re.compile(r'([1-9][0-9]*|0)\.[0-9]{1,2}$')
 #
 def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1):
     ValueList = []
     Last = 0
     Escaped = False
-    InString = False
+    InSingleQuoteString = False
+    InDoubleQuoteString = False
     InParenthesis = 0
     for Index in range(0, len(String)):
         Char = String[Index]
 
         if not Escaped:
             # Found a splitter not in a string, split it
-            if not InString and InParenthesis == 0 and Char == SplitTag:
+            if (not InSingleQuoteString or not InDoubleQuoteString) and InParenthesis == 0 and Char == SplitTag:
                 ValueList.append(String[Last:Index].strip())
                 Last = Index + 1
                 if MaxSplit > 0 and len(ValueList) >= MaxSplit:
                     break
 
-            if Char == '\\' and InString:
+            if Char == '\\' and (InSingleQuoteString or InDoubleQuoteString):
                 Escaped = True
-            elif Char == '"':
-                if not InString:
-                    InString = True
+            elif Char == '"' and not InSingleQuoteString:
+                if not InDoubleQuoteString:
+                    InDoubleQuoteString = True
+                else:
+                    InDoubleQuoteString = False
+            elif Char == "'" and not InDoubleQuoteString:
+                if not InSingleQuoteString:
+                    InSingleQuoteString = True
                 else:
-                    InString = False
+                    InSingleQuoteString = False
             elif Char == '(':
                 InParenthesis = InParenthesis + 1
             elif Char == ')':
                 InParenthesis = InParenthesis - 1
         else:
@@ -400,19 +406,22 @@ def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyl
     if AllowCppStyleComment:
         Line = Line.replace(DataType.TAB_COMMENT_EDK_SPLIT, CommentCharacter)
     #
     # separate comments and statements, but we should escape comment character in string
     #
-    InString = False
+    InDoubleQuoteString = False
+    InSingleQuoteString = False
     CommentInString = False
     Comment = ''
     for Index in range(0, len(Line)):
-        if Line[Index] == '"':
-            InString = not InString
-        elif Line[Index] == CommentCharacter and InString:
+        if Line[Index] == '"' and not InSingleQuoteString:
+            InDoubleQuoteString = not InDoubleQuoteString
+        elif Line[Index] == "'" and not InDoubleQuoteString:
+            InSingleQuoteString = not InSingleQuoteString
+        elif Line[Index] == CommentCharacter and (InDoubleQuoteString or InSingleQuoteString):
             CommentInString = True
-        elif Line[Index] == CommentCharacter and not InString:
+        elif Line[Index] == CommentCharacter and not (InDoubleQuoteString or InSingleQuoteString):
             Comment = Line[Index:].strip()
             Line = Line[0:Index].strip()
             break
 
     return Line, Comment
diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index 75b877a5aa..ba3a8b1457 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -989,10 +989,12 @@ class DscBuildData(PlatformBuildClassObject):
                         FoundFlag = True
                 if FieldName:
                     NewValue = self.GetFieldValueFromComm(pcdvalue, TokenSpaceGuidCName, TokenCName, FieldName)
                     GlobalData.BuildOptionPcd[i] = (TokenSpaceGuidCName, TokenCName, FieldName,NewValue,("build command options",1))
                 else:
+                    # Replace \' to ', \\\' to \'
+                    pcdvalue = pcdvalue.replace("\\\\\\'", '\\\\\\"').replace('\\\'', '\'').replace('\\\\\\"', "\\'")
                     for key in self.DecPcds:
                         PcdItem = self.DecPcds[key]
                         if HasTokenSpace:
                             if (PcdItem.TokenCName, PcdItem.TokenSpaceGuidCName) == (TokenCName, TokenSpaceGuidCName):
                                 PcdDatumType = PcdItem.DatumType
@@ -1000,28 +1002,25 @@ class DscBuildData(PlatformBuildClassObject):
                                     try:
                                         pcdvalue = ValueExpressionEx(pcdvalue[1:], PcdDatumType, self._GuidDict)(True)
                                     except BadExpression, Value:
                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %
                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
-                                    if PcdDatumType == "VOID*":
-                                        pcdvalue = 'H' + pcdvalue
+                                    pcdvalue = 'H' + pcdvalue
                                 elif pcdvalue.startswith("L'"):
                                     try:
                                         pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)
                                     except BadExpression, Value:
                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %
                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
-                                    if pcdvalue.startswith('{'):
-                                        pcdvalue = 'H' + pcdvalue
+                                    pcdvalue = 'H' + pcdvalue
                                 elif pcdvalue.startswith("'"):
                                     try:
                                         pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)
                                     except BadExpression, Value:
                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %
                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
-                                    if pcdvalue.startswith('{'):
-                                        pcdvalue = 'H' + pcdvalue
+                                    pcdvalue = 'H' + pcdvalue
                                 elif pcdvalue.startswith('L'):
                                     pcdvalue = 'L"' + pcdvalue[1:] + '"'
                                     try:
                                         pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)
                                     except BadExpression, Value:
@@ -1029,12 +1028,16 @@ class DscBuildData(PlatformBuildClassObject):
                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
                                 else:
                                     try:
                                         pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)
                                     except BadExpression, Value:
-                                        EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %
-                                                        (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
+                                        try:
+                                            pcdvalue = '"' + pcdvalue + '"'
+                                            pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)
+                                        except BadExpression, Value:
+                                            EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %
+                                                            (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
                                 NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue)
                                 FoundFlag = True
                         else:
                             if PcdItem.TokenCName == TokenCName:
                                 if not PcdItem.TokenSpaceGuidCName in TokenSpaceGuidCNameList:
@@ -1046,30 +1049,27 @@ class DscBuildData(PlatformBuildClassObject):
                                             try:
                                                 pcdvalue = ValueExpressionEx(pcdvalue[1:], PcdDatumType, self._GuidDict)(True)
                                             except BadExpression, Value:
                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' %
                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
-                                            if PcdDatumType == "VOID*":
-                                                pcdvalue = 'H' + pcdvalue
+                                            pcdvalue = 'H' + pcdvalue
                                         elif pcdvalue.startswith("L'"):
                                             try:
                                                 pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(
                                                     True)
                                             except BadExpression, Value:
                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %
                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
-                                            if pcdvalue.startswith('{'):
-                                                pcdvalue = 'H' + pcdvalue
+                                            pcdvalue = 'H' + pcdvalue
                                         elif pcdvalue.startswith("'"):
                                             try:
                                                 pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(
                                                     True)
                                             except BadExpression, Value:
                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %
                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
-                                            if pcdvalue.startswith('{'):
-                                                pcdvalue = 'H' + pcdvalue
+                                            pcdvalue = 'H' + pcdvalue
                                         elif pcdvalue.startswith('L'):
                                             pcdvalue = 'L"' + pcdvalue[1:] + '"'
                                             try:
                                                 pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(
                                                     True)
@@ -1078,13 +1078,16 @@ class DscBuildData(PlatformBuildClassObject):
                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
                                         else:
                                             try:
                                                 pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)
                                             except BadExpression, Value:
-                                                EdkLogger.error('Parser', FORMAT_INVALID,
-                                                                'PCD [%s.%s] Value "%s",  %s' %
-                                                                (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
+                                                try:
+                                                    pcdvalue = '"' + pcdvalue + '"'
+                                                    pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)
+                                                except BadExpression, Value:
+                                                    EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %
+                                                                    (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))
                                         NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue)
                                         FoundFlag = True
                                     else:
                                         EdkLogger.error(
                                                 'build',
-- 
2.12.2.windows.2

_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel