1## @file
2# preprocess source file
3#
4#  Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>
5#
6#  This program and the accompanying materials
7#  are licensed and made available under the terms and conditions of the BSD License
8#  which accompanies this distribution.  The full text of the license may be found at
9#  http://opensource.org/licenses/bsd-license.php
10#
11#  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12#  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
13#
14
15##
16# Import Modules
17#
18import sys
19import Common.LongFilePathOs as os
20import re
21import CodeFragmentCollector
22import FileProfile
23from CommonDataClass import DataClass
24from Common import EdkLogger
25from EotToolError import *
26import EotGlobalData
27
28# Global Dicts
29IncludeFileListDict = {}
30IncludePathListDict = {}
31ComplexTypeDict = {}
32SUDict = {}
33
34## GetFuncDeclPattern() method
35#
36#  Get the pattern of function declaration
37#
38#  @return p:    the pattern of function declaration
39#
40def GetFuncDeclPattern():
41    p = re.compile(r'(EFIAPI|EFI_BOOT_SERVICE|EFI_RUNTIME_SERVICE)?\s*[_\w]+\s*\(.*\).*', re.DOTALL)
42    return p
43
44## GetArrayPattern() method
45#
46#  Get the pattern of array
47#
48#  @return p:    the pattern of array
49#
50def GetArrayPattern():
51    p = re.compile(r'[_\w]*\s*[\[.*\]]+')
52    return p
53
54## GetTypedefFuncPointerPattern() method
55#
56#  Get the pattern of function pointer
57#
58#  @return p:    the pattern of function pointer
59#
60def GetTypedefFuncPointerPattern():
61    p = re.compile('[_\w\s]*\([\w\s]*\*+\s*[_\w]+\s*\)\s*\(.*\)', re.DOTALL)
62    return p
63
64## GetDB() method
65#
66#  Get global database instance
67#
68#  @return EotGlobalData.gDb:    the global database instance
69#
70def GetDB():
71    return EotGlobalData.gDb
72
73## PrintErrorMsg() method
74#
75#  print error message
76#
77#  @param ErrorType: Type of error
78#  @param Msg: Error message
79#  @param TableName: table name of error found
80#  @param ItemId: id of item
81#
82def PrintErrorMsg(ErrorType, Msg, TableName, ItemId):
83    Msg = Msg.replace('\n', '').replace('\r', '')
84    MsgPartList = Msg.split()
85    Msg = ''
86    for Part in MsgPartList:
87        Msg += Part
88        Msg += ' '
89    GetDB().TblReport.Insert(ErrorType, OtherMsg = Msg, BelongsToTable = TableName, BelongsToItem = ItemId)
90
91## GetIdType() method
92#
93#  Find type of input string
94#
95#  @param Str: String to be parsed
96#
97#  @return Type: The type of the string
98#
99def GetIdType(Str):
100    Type = DataClass.MODEL_UNKNOWN
101    Str = Str.replace('#', '# ')
102    List = Str.split()
103    if List[1] == 'include':
104        Type = DataClass.MODEL_IDENTIFIER_INCLUDE
105    elif List[1] == 'define':
106        Type = DataClass.MODEL_IDENTIFIER_MACRO_DEFINE
107    elif List[1] == 'ifdef':
108        Type = DataClass.MODEL_IDENTIFIER_MACRO_IFDEF
109    elif List[1] == 'ifndef':
110        Type = DataClass.MODEL_IDENTIFIER_MACRO_IFNDEF
111    elif List[1] == 'endif':
112        Type = DataClass.MODEL_IDENTIFIER_MACRO_ENDIF
113    elif List[1] == 'pragma':
114        Type = DataClass.MODEL_IDENTIFIER_MACRO_PROGMA
115    else:
116        Type = DataClass.MODEL_UNKNOWN
117    return Type
118
119## GetIdentifierList() method
120#
121#  Get id of all files
122#
123#  @return IdList: The list of all id of files
124#
125def GetIdentifierList():
126    IdList = []
127
128    for pp in FileProfile.PPDirectiveList:
129        Type = GetIdType(pp.Content)
130        IdPP = DataClass.IdentifierClass(-1, '', '', '', pp.Content, Type, -1, -1, pp.StartPos[0],pp.StartPos[1],pp.EndPos[0],pp.EndPos[1])
131        IdList.append(IdPP)
132
133    for ae in FileProfile.AssignmentExpressionList:
134        IdAE = DataClass.IdentifierClass(-1, ae.Operator, '', ae.Name, ae.Value, DataClass.MODEL_IDENTIFIER_ASSIGNMENT_EXPRESSION, -1, -1, ae.StartPos[0],ae.StartPos[1],ae.EndPos[0],ae.EndPos[1])
135        IdList.append(IdAE)
136
137    FuncDeclPattern = GetFuncDeclPattern()
138    ArrayPattern = GetArrayPattern()
139    for var in FileProfile.VariableDeclarationList:
140        DeclText = var.Declarator.strip()
141        while DeclText.startswith('*'):
142            var.Modifier += '*'
143            DeclText = DeclText.lstrip('*').strip()
144        var.Declarator = DeclText
145        if FuncDeclPattern.match(var.Declarator):
146            DeclSplitList = var.Declarator.split('(')
147            FuncName = DeclSplitList[0]
148            FuncNamePartList = FuncName.split()
149            if len(FuncNamePartList) > 1:
150                FuncName = FuncNamePartList[-1]
151                Index = 0
152                while Index < len(FuncNamePartList) - 1:
153                    var.Modifier += ' ' + FuncNamePartList[Index]
154                    var.Declarator = var.Declarator.lstrip().lstrip(FuncNamePartList[Index])
155                    Index += 1
156            IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', var.Declarator, '', DataClass.MODEL_IDENTIFIER_FUNCTION_DECLARATION, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1])
157            IdList.append(IdVar)
158            continue
159
160        if var.Declarator.find('{') == -1:
161            for decl in var.Declarator.split(','):
162                DeclList = decl.split('=')
163                Name = DeclList[0].strip()
164                if ArrayPattern.match(Name):
165                    LSBPos = var.Declarator.find('[')
166                    var.Modifier += ' ' + Name[LSBPos:]
167                    Name = Name[0:LSBPos]
168
169                IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', Name, (len(DeclList) > 1 and [DeclList[1]]or [''])[0], DataClass.MODEL_IDENTIFIER_VARIABLE, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1])
170                IdList.append(IdVar)
171        else:
172            DeclList = var.Declarator.split('=')
173            Name = DeclList[0].strip()
174            if ArrayPattern.match(Name):
175                LSBPos = var.Declarator.find('[')
176                var.Modifier += ' ' + Name[LSBPos:]
177                Name = Name[0:LSBPos]
178            IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', Name, (len(DeclList) > 1 and [DeclList[1]]or [''])[0], DataClass.MODEL_IDENTIFIER_VARIABLE, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1])
179            IdList.append(IdVar)
180
181    for enum in FileProfile.EnumerationDefinitionList:
182        LBPos = enum.Content.find('{')
183        RBPos = enum.Content.find('}')
184        Name = enum.Content[4:LBPos].strip()
185        Value = enum.Content[LBPos+1:RBPos]
186        IdEnum = DataClass.IdentifierClass(-1, '', '', Name, Value, DataClass.MODEL_IDENTIFIER_ENUMERATE, -1, -1, enum.StartPos[0],enum.StartPos[1],enum.EndPos[0],enum.EndPos[1])
187        IdList.append(IdEnum)
188
189    for su in FileProfile.StructUnionDefinitionList:
190        Type = DataClass.MODEL_IDENTIFIER_STRUCTURE
191        SkipLen = 6
192        if su.Content.startswith('union'):
193            Type = DataClass.MODEL_IDENTIFIER_UNION
194            SkipLen = 5
195        LBPos = su.Content.find('{')
196        RBPos = su.Content.find('}')
197        if LBPos == -1 or RBPos == -1:
198            Name = su.Content[SkipLen:].strip()
199            Value = ''
200        else:
201            Name = su.Content[SkipLen:LBPos].strip()
202            Value = su.Content[LBPos+1:RBPos]
203        IdPE = DataClass.IdentifierClass(-1, '', '', Name, Value, Type, -1, -1, su.StartPos[0],su.StartPos[1],su.EndPos[0],su.EndPos[1])
204        IdList.append(IdPE)
205
206    TdFuncPointerPattern = GetTypedefFuncPointerPattern()
207    for td in FileProfile.TypedefDefinitionList:
208        Modifier = ''
209        Name = td.ToType
210        Value = td.FromType
211        if TdFuncPointerPattern.match(td.ToType):
212            Modifier = td.FromType
213            LBPos = td.ToType.find('(')
214            TmpStr = td.ToType[LBPos+1:].strip()
215            StarPos = TmpStr.find('*')
216            if StarPos != -1:
217                Modifier += ' ' + TmpStr[0:StarPos]
218            while TmpStr[StarPos] == '*':
219                Modifier += ' ' + '*'
220                StarPos += 1
221            TmpStr = TmpStr[StarPos:].strip()
222            RBPos = TmpStr.find(')')
223            Name = TmpStr[0:RBPos]
224            Value = 'FP' + TmpStr[RBPos + 1:]
225
226        IdTd = DataClass.IdentifierClass(-1, Modifier, '', Name, Value, DataClass.MODEL_IDENTIFIER_TYPEDEF, -1, -1, td.StartPos[0],td.StartPos[1],td.EndPos[0],td.EndPos[1])
227        IdList.append(IdTd)
228
229    for funcCall in FileProfile.FunctionCallingList:
230        IdFC = DataClass.IdentifierClass(-1, '', '', funcCall.FuncName, funcCall.ParamList, DataClass.MODEL_IDENTIFIER_FUNCTION_CALLING, -1, -1, funcCall.StartPos[0],funcCall.StartPos[1],funcCall.EndPos[0],funcCall.EndPos[1])
231        IdList.append(IdFC)
232    return IdList
233
234## GetParamList() method
235#
236#  Get a list of parameters
237#
238#  @param FuncDeclarator: Function declarator
239#  @param FuncNameLine: Line number of function name
240#  @param FuncNameOffset: Offset of function name
241#
242#  @return ParamIdList: A list of parameters
243#
244def GetParamList(FuncDeclarator, FuncNameLine = 0, FuncNameOffset = 0):
245    ParamIdList = []
246    DeclSplitList = FuncDeclarator.split('(')
247    if len(DeclSplitList) < 2:
248        return ParamIdList
249    FuncName = DeclSplitList[0]
250    ParamStr = DeclSplitList[1].rstrip(')')
251    LineSkipped = 0
252    OffsetSkipped = 0
253    Start = 0
254    while FuncName.find('\n', Start) != -1:
255        LineSkipped += 1
256        OffsetSkipped = 0
257        Start += FuncName.find('\n', Start)
258        Start += 1
259    OffsetSkipped += len(FuncName[Start:])
260    OffsetSkipped += 1 #skip '('
261    ParamBeginLine = FuncNameLine + LineSkipped
262    ParamBeginOffset = OffsetSkipped
263    for p in ParamStr.split(','):
264        ListP = p.split()
265        if len(ListP) == 0:
266            continue
267        ParamName = ListP[-1]
268        DeclText = ParamName.strip()
269        RightSpacePos = p.rfind(ParamName)
270        ParamModifier = p[0:RightSpacePos]
271        if ParamName == 'OPTIONAL':
272            if ParamModifier == '':
273                ParamModifier += ' ' + 'OPTIONAL'
274                DeclText = ''
275            else:
276                ParamName = ListP[-2]
277                DeclText = ParamName.strip()
278                RightSpacePos = p.rfind(ParamName)
279                ParamModifier = p[0:RightSpacePos]
280                ParamModifier += 'OPTIONAL'
281        while DeclText.startswith('*'):
282            ParamModifier += ' ' + '*'
283            DeclText = DeclText.lstrip('*').strip()
284        ParamName = DeclText
285
286        Start = 0
287        while p.find('\n', Start) != -1:
288            LineSkipped += 1
289            OffsetSkipped = 0
290            Start += p.find('\n', Start)
291            Start += 1
292        OffsetSkipped += len(p[Start:])
293
294        ParamEndLine = ParamBeginLine + LineSkipped
295        ParamEndOffset = OffsetSkipped
296        IdParam = DataClass.IdentifierClass(-1, ParamModifier, '', ParamName, '', DataClass.MODEL_IDENTIFIER_PARAMETER, -1, -1, ParamBeginLine, ParamBeginOffset, ParamEndLine, ParamEndOffset)
297        ParamIdList.append(IdParam)
298        ParamBeginLine = ParamEndLine
299        ParamBeginOffset = OffsetSkipped + 1 #skip ','
300
301    return ParamIdList
302
303## GetFunctionList()
304#
305#  Get a list of functions
306#
307#  @return FuncObjList: A list of function objects
308#
309def GetFunctionList():
310    FuncObjList = []
311    for FuncDef in FileProfile.FunctionDefinitionList:
312        ParamIdList = []
313        DeclText = FuncDef.Declarator.strip()
314        while DeclText.startswith('*'):
315            FuncDef.Modifier += '*'
316            DeclText = DeclText.lstrip('*').strip()
317
318        FuncDef.Declarator = FuncDef.Declarator.lstrip('*')
319        DeclSplitList = FuncDef.Declarator.split('(')
320        if len(DeclSplitList) < 2:
321            continue
322
323        FuncName = DeclSplitList[0]
324        FuncNamePartList = FuncName.split()
325        if len(FuncNamePartList) > 1:
326            FuncName = FuncNamePartList[-1]
327            Index = 0
328            while Index < len(FuncNamePartList) - 1:
329                FuncDef.Modifier += ' ' + FuncNamePartList[Index]
330                Index += 1
331
332        FuncObj = DataClass.FunctionClass(-1, FuncDef.Declarator, FuncDef.Modifier, FuncName.strip(), '', FuncDef.StartPos[0],FuncDef.StartPos[1],FuncDef.EndPos[0],FuncDef.EndPos[1], FuncDef.LeftBracePos[0], FuncDef.LeftBracePos[1], -1, ParamIdList, [])
333        FuncObjList.append(FuncObj)
334
335    return FuncObjList
336
337## CreateCCodeDB() method
338#
339#  Create database for all c code
340#
341#  @param FileNameList: A list of all c code file names
342#
343def CreateCCodeDB(FileNameList):
344    FileObjList = []
345    ParseErrorFileList = []
346    ParsedFiles = {}
347    for FullName in FileNameList:
348        if os.path.splitext(FullName)[1] in ('.h', '.c'):
349            if FullName.lower() in ParsedFiles:
350                continue
351            ParsedFiles[FullName.lower()] = 1
352            EdkLogger.info("Parsing " + FullName)
353            model = FullName.endswith('c') and DataClass.MODEL_FILE_C or DataClass.MODEL_FILE_H
354            collector = CodeFragmentCollector.CodeFragmentCollector(FullName)
355            try:
356                collector.ParseFile()
357            except:
358                ParseErrorFileList.append(FullName)
359            BaseName = os.path.basename(FullName)
360            DirName = os.path.dirname(FullName)
361            Ext = os.path.splitext(BaseName)[1].lstrip('.')
362            ModifiedTime = os.path.getmtime(FullName)
363            FileObj = DataClass.FileClass(-1, BaseName, Ext, DirName, FullName, model, ModifiedTime, GetFunctionList(), GetIdentifierList(), [])
364            FileObjList.append(FileObj)
365            collector.CleanFileProfileBuffer()
366
367    if len(ParseErrorFileList) > 0:
368        EdkLogger.info("Found unrecoverable error during parsing:\n\t%s\n" % "\n\t".join(ParseErrorFileList))
369
370    Db = EotGlobalData.gDb
371    for file in FileObjList:
372        Db.InsertOneFile(file)
373
374    Db.UpdateIdentifierBelongsToFunction()
375
376##
377#
378# This acts like the main() function for the script, unless it is 'import'ed into another
379# script.
380#
381if __name__ == '__main__':
382
383    EdkLogger.Initialize()
384    EdkLogger.SetLevel(EdkLogger.QUIET)
385    CollectSourceCodeDataIntoDB(sys.argv[1])
386
387    print 'Done!'
388