1## @file
2# This file is used to be the main entrance of ECC tool
3#
4# Copyright (c) 2009 - 2016, Intel Corporation. All rights reserved.<BR>
5# This program and the accompanying materials
6# are licensed and made available under the terms and conditions of the BSD License
7# which accompanies this distribution.  The full text of the license may be found at
8# http://opensource.org/licenses/bsd-license.php
9#
10# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12#
13
14##
15# Import Modules
16#
17import Common.LongFilePathOs as os, time, glob, sys
18import Common.EdkLogger as EdkLogger
19import Database
20import EccGlobalData
21from MetaDataParser import *
22from optparse import OptionParser
23from Configuration import Configuration
24from Check import Check
25import Common.GlobalData as GlobalData
26
27from Common.String import NormPath
28from Common.BuildVersion import gBUILD_VERSION
29from Common import BuildToolError
30from Common.Misc import PathClass
31from Common.Misc import DirCache
32from MetaFileWorkspace.MetaFileParser import DscParser
33from MetaFileWorkspace.MetaFileParser import DecParser
34from MetaFileWorkspace.MetaFileParser import InfParser
35from MetaFileWorkspace.MetaFileParser import Fdf
36from MetaFileWorkspace.MetaFileTable import MetaFileStorage
37import c
38import re, string
39from Exception import *
40from Common.LongFilePathSupport import OpenLongFilePath as open
41from Common.MultipleWorkspace import MultipleWorkspace as mws
42
43## Ecc
44#
45# This class is used to define Ecc main entrance
46#
47# @param object:          Inherited from object class
48#
49class Ecc(object):
50    def __init__(self):
51        # Version and Copyright
52        self.VersionNumber = ("1.0" + " Build " + gBUILD_VERSION)
53        self.Version = "%prog Version " + self.VersionNumber
54        self.Copyright = "Copyright (c) 2009 - 2016, Intel Corporation  All rights reserved."
55
56        self.InitDefaultConfigIni()
57        self.OutputFile = 'output.txt'
58        self.ReportFile = 'Report.csv'
59        self.ExceptionFile = 'exception.xml'
60        self.IsInit = True
61        self.ScanSourceCode = True
62        self.ScanMetaData = True
63        self.MetaFile = ''
64        self.OnlyScan = None
65
66        # Parse the options and args
67        self.ParseOption()
68        EdkLogger.info(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[00:00]" + "\n")
69
70        #
71        # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
72        #
73        WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
74        os.environ["WORKSPACE"] = WorkspaceDir
75
76        # set multiple workspace
77        PackagesPath = os.getenv("PACKAGES_PATH")
78        mws.setWs(WorkspaceDir, PackagesPath)
79
80        if "ECP_SOURCE" not in os.environ:
81            os.environ["ECP_SOURCE"] = mws.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
82        if "EFI_SOURCE" not in os.environ:
83            os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
84        if "EDK_SOURCE" not in os.environ:
85            os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
86
87        #
88        # Unify case of characters on case-insensitive systems
89        #
90        EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
91        EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
92        EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
93
94        os.environ["EFI_SOURCE"] = EfiSourceDir
95        os.environ["EDK_SOURCE"] = EdkSourceDir
96        os.environ["ECP_SOURCE"] = EcpSourceDir
97
98        GlobalData.gWorkspace = WorkspaceDir
99        GlobalData.gEfiSource = EfiSourceDir
100        GlobalData.gEdkSource = EdkSourceDir
101        GlobalData.gEcpSource = EcpSourceDir
102
103        GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
104        GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
105        GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
106        GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
107
108        EdkLogger.info("Loading ECC configuration ... done")
109        # Generate checkpoints list
110        EccGlobalData.gConfig = Configuration(self.ConfigFile)
111
112        # Generate exception list
113        EccGlobalData.gException = ExceptionCheck(self.ExceptionFile)
114
115        # Init Ecc database
116        EccGlobalData.gDb = Database.Database(Database.DATABASE_PATH)
117        EccGlobalData.gDb.InitDatabase(self.IsInit)
118
119        #
120        # Get files real name in workspace dir
121        #
122        GlobalData.gAllFiles = DirCache(GlobalData.gWorkspace)
123
124        # Build ECC database
125#         self.BuildDatabase()
126        self.DetectOnlyScanDirs()
127
128        # Start to check
129        self.Check()
130
131        # Show report
132        self.GenReport()
133
134        # Close Database
135        EccGlobalData.gDb.Close()
136
137    def InitDefaultConfigIni(self):
138        paths = map(lambda p: os.path.join(p, 'Ecc', 'config.ini'), sys.path)
139        paths = (os.path.realpath('config.ini'),) + tuple(paths)
140        for path in paths:
141            if os.path.exists(path):
142                self.ConfigFile = path
143                return
144        self.ConfigFile = 'config.ini'
145
146
147    ## DetectOnlyScan
148    #
149    # Detect whether only scanned folders have been enabled
150    #
151    def DetectOnlyScanDirs(self):
152        if self.OnlyScan == True:
153            OnlyScanDirs = []
154            # Use regex here if multiple spaces or TAB exists in ScanOnlyDirList in config.ini file
155            for folder in re.finditer(r'\S+', EccGlobalData.gConfig.ScanOnlyDirList):
156                OnlyScanDirs.append(folder.group())
157            if len(OnlyScanDirs) != 0:
158                self.BuildDatabase(OnlyScanDirs)
159            else:
160                EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Use -f option need to fill specific folders in config.ini file")
161        else:
162            self.BuildDatabase()
163
164
165    ## BuildDatabase
166    #
167    # Build the database for target
168    #
169    def BuildDatabase(self, SpeciDirs = None):
170        # Clean report table
171        EccGlobalData.gDb.TblReport.Drop()
172        EccGlobalData.gDb.TblReport.Create()
173
174        # Build database
175        if self.IsInit:
176            if self.ScanMetaData:
177                EdkLogger.quiet("Building database for Meta Data File ...")
178                self.BuildMetaDataFileDatabase(SpeciDirs)
179            if self.ScanSourceCode:
180                EdkLogger.quiet("Building database for Meta Data File Done!")
181                if SpeciDirs == None:
182                    c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
183                else:
184                    for specificDir in SpeciDirs:
185                        c.CollectSourceCodeDataIntoDB(os.path.join(EccGlobalData.gTarget, specificDir))
186
187        EccGlobalData.gIdentifierTableList = GetTableList((MODEL_FILE_C, MODEL_FILE_H), 'Identifier', EccGlobalData.gDb)
188        EccGlobalData.gCFileList = GetFileList(MODEL_FILE_C, EccGlobalData.gDb)
189        EccGlobalData.gHFileList = GetFileList(MODEL_FILE_H, EccGlobalData.gDb)
190        EccGlobalData.gUFileList = GetFileList(MODEL_FILE_UNI, EccGlobalData.gDb)
191
192    ## BuildMetaDataFileDatabase
193    #
194    # Build the database for meta data files
195    #
196    def BuildMetaDataFileDatabase(self, SpecificDirs = None):
197        ScanFolders = []
198        if SpecificDirs == None:
199            ScanFolders.append(EccGlobalData.gTarget)
200        else:
201            for specificDir in SpecificDirs:
202                ScanFolders.append(os.path.join(EccGlobalData.gTarget, specificDir))
203        EdkLogger.quiet("Building database for meta data files ...")
204        Op = open(EccGlobalData.gConfig.MetaDataFileCheckPathOfGenerateFileList, 'w+')
205        #SkipDirs = Read from config file
206        SkipDirs = EccGlobalData.gConfig.SkipDirList
207        SkipDirString = string.join(SkipDirs, '|')
208#         p = re.compile(r'.*[\\/](?:%s)[\\/]?.*' % SkipDirString)
209        p = re.compile(r'.*[\\/](?:%s^\S)[\\/]?.*' % SkipDirString)
210        for scanFolder in ScanFolders:
211            for Root, Dirs, Files in os.walk(scanFolder):
212                if p.match(Root.upper()):
213                    continue
214                for Dir in Dirs:
215                    Dirname = os.path.join(Root, Dir)
216                    if os.path.islink(Dirname):
217                        Dirname = os.path.realpath(Dirname)
218                        if os.path.isdir(Dirname):
219                            # symlinks to directories are treated as directories
220                            Dirs.remove(Dir)
221                            Dirs.append(Dirname)
222
223                for File in Files:
224                    if len(File) > 4 and File[-4:].upper() == ".DEC":
225                        Filename = os.path.normpath(os.path.join(Root, File))
226                        EdkLogger.quiet("Parsing %s" % Filename)
227                        Op.write("%s\r" % Filename)
228                        #Dec(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
229                        self.MetaFile = DecParser(Filename, MODEL_FILE_DEC, EccGlobalData.gDb.TblDec)
230                        self.MetaFile.Start()
231                        continue
232                    if len(File) > 4 and File[-4:].upper() == ".DSC":
233                        Filename = os.path.normpath(os.path.join(Root, File))
234                        EdkLogger.quiet("Parsing %s" % Filename)
235                        Op.write("%s\r" % Filename)
236                        #Dsc(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
237                        self.MetaFile = DscParser(PathClass(Filename, Root), MODEL_FILE_DSC, MetaFileStorage(EccGlobalData.gDb.TblDsc.Cur, Filename, MODEL_FILE_DSC, True))
238                        # alwasy do post-process, in case of macros change
239                        self.MetaFile.DoPostProcess()
240                        self.MetaFile.Start()
241                        self.MetaFile._PostProcess()
242                        continue
243                    if len(File) > 4 and File[-4:].upper() == ".INF":
244                        Filename = os.path.normpath(os.path.join(Root, File))
245                        EdkLogger.quiet("Parsing %s" % Filename)
246                        Op.write("%s\r" % Filename)
247                        #Inf(Filename, True, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
248                        self.MetaFile = InfParser(Filename, MODEL_FILE_INF, EccGlobalData.gDb.TblInf)
249                        self.MetaFile.Start()
250                        continue
251                    if len(File) > 4 and File[-4:].upper() == ".FDF":
252                        Filename = os.path.normpath(os.path.join(Root, File))
253                        EdkLogger.quiet("Parsing %s" % Filename)
254                        Op.write("%s\r" % Filename)
255                        Fdf(Filename, True, EccGlobalData.gWorkspace, EccGlobalData.gDb)
256                        continue
257                    if len(File) > 4 and File[-4:].upper() == ".UNI":
258                        Filename = os.path.normpath(os.path.join(Root, File))
259                        EdkLogger.quiet("Parsing %s" % Filename)
260                        Op.write("%s\r" % Filename)
261                        FileID = EccGlobalData.gDb.TblFile.InsertFile(Filename, MODEL_FILE_UNI)
262                        EccGlobalData.gDb.TblReport.UpdateBelongsToItemByFile(FileID, File)
263                        continue
264
265        Op.close()
266
267        # Commit to database
268        EccGlobalData.gDb.Conn.commit()
269
270        EdkLogger.quiet("Building database for meta data files done!")
271
272    ##
273    #
274    # Check each checkpoint
275    #
276    def Check(self):
277        EdkLogger.quiet("Checking ...")
278        EccCheck = Check()
279        EccCheck.Check()
280        EdkLogger.quiet("Checking  done!")
281
282    ##
283    #
284    # Generate the scan report
285    #
286    def GenReport(self):
287        EdkLogger.quiet("Generating report ...")
288        EccGlobalData.gDb.TblReport.ToCSV(self.ReportFile)
289        EdkLogger.quiet("Generating report done!")
290
291    def GetRealPathCase(self, path):
292        TmpPath = path.rstrip(os.sep)
293        PathParts = TmpPath.split(os.sep)
294        if len(PathParts) == 0:
295            return path
296        if len(PathParts) == 1:
297            if PathParts[0].strip().endswith(':'):
298                return PathParts[0].upper()
299            # Relative dir, list . current dir
300            Dirs = os.listdir('.')
301            for Dir in Dirs:
302                if Dir.upper() == PathParts[0].upper():
303                    return Dir
304
305        if PathParts[0].strip().endswith(':'):
306            PathParts[0] = PathParts[0].upper()
307        ParentDir = PathParts[0]
308        RealPath = ParentDir
309        if PathParts[0] == '':
310            RealPath = os.sep
311            ParentDir = os.sep
312
313        PathParts.remove(PathParts[0])    # need to remove the parent
314        for Part in PathParts:
315            Dirs = os.listdir(ParentDir + os.sep)
316            for Dir in Dirs:
317                if Dir.upper() == Part.upper():
318                    RealPath += os.sep
319                    RealPath += Dir
320                    break
321            ParentDir += os.sep
322            ParentDir += Dir
323
324        return RealPath
325
326    ## ParseOption
327    #
328    # Parse options
329    #
330    def ParseOption(self):
331        (Options, Target) = self.EccOptionParser()
332
333        if Options.Workspace:
334            os.environ["WORKSPACE"] = Options.Workspace
335
336        # Check workspace envirnoment
337        if "WORKSPACE" not in os.environ:
338            EdkLogger.error("ECC", BuildToolError.ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
339                            ExtraData="WORKSPACE")
340        else:
341            EccGlobalData.gWorkspace = os.path.normpath(os.getenv("WORKSPACE"))
342            if not os.path.exists(EccGlobalData.gWorkspace):
343                EdkLogger.error("ECC", BuildToolError.FILE_NOT_FOUND, ExtraData="WORKSPACE = %s" % EccGlobalData.gWorkspace)
344            os.environ["WORKSPACE"] = EccGlobalData.gWorkspace
345        # Set log level
346        self.SetLogLevel(Options)
347
348        # Set other options
349        if Options.ConfigFile != None:
350            self.ConfigFile = Options.ConfigFile
351        if Options.OutputFile != None:
352            self.OutputFile = Options.OutputFile
353        if Options.ReportFile != None:
354            self.ReportFile = Options.ReportFile
355        if Options.ExceptionFile != None:
356            self.ExceptionFile = Options.ExceptionFile
357        if Options.Target != None:
358            if not os.path.isdir(Options.Target):
359                EdkLogger.error("ECC", BuildToolError.OPTION_VALUE_INVALID, ExtraData="Target [%s] does NOT exist" % Options.Target)
360            else:
361                EccGlobalData.gTarget = self.GetRealPathCase(os.path.normpath(Options.Target))
362        else:
363            EdkLogger.warn("Ecc", EdkLogger.ECC_ERROR, "The target source tree was not specified, using current WORKSPACE instead!")
364            EccGlobalData.gTarget = os.path.normpath(os.getenv("WORKSPACE"))
365        if Options.keepdatabase != None:
366            self.IsInit = False
367        if Options.metadata != None and Options.sourcecode != None:
368            EdkLogger.error("ECC", BuildToolError.OPTION_CONFLICT, ExtraData="-m and -s can't be specified at one time")
369        if Options.metadata != None:
370            self.ScanSourceCode = False
371        if Options.sourcecode != None:
372            self.ScanMetaData = False
373        if Options.folders != None:
374            self.OnlyScan = True
375
376    ## SetLogLevel
377    #
378    # Set current log level of the tool based on args
379    #
380    # @param Option:  The option list including log level setting
381    #
382    def SetLogLevel(self, Option):
383        if Option.verbose != None:
384            EdkLogger.SetLevel(EdkLogger.VERBOSE)
385        elif Option.quiet != None:
386            EdkLogger.SetLevel(EdkLogger.QUIET)
387        elif Option.debug != None:
388            EdkLogger.SetLevel(Option.debug + 1)
389        else:
390            EdkLogger.SetLevel(EdkLogger.INFO)
391
392    ## Parse command line options
393    #
394    # Using standard Python module optparse to parse command line option of this tool.
395    #
396    # @retval Opt   A optparse.Values object containing the parsed options
397    # @retval Args  Target of build command
398    #
399    def EccOptionParser(self):
400        Parser = OptionParser(description = self.Copyright, version = self.Version, prog = "Ecc.exe", usage = "%prog [options]")
401        Parser.add_option("-t", "--target sourcepath", action="store", type="string", dest='Target',
402            help="Check all files under the target workspace.")
403        Parser.add_option("-c", "--config filename", action="store", type="string", dest="ConfigFile",
404            help="Specify a configuration file. Defaultly use config.ini under ECC tool directory.")
405        Parser.add_option("-o", "--outfile filename", action="store", type="string", dest="OutputFile",
406            help="Specify the name of an output file, if and only if one filename was specified.")
407        Parser.add_option("-r", "--reportfile filename", action="store", type="string", dest="ReportFile",
408            help="Specify the name of an report file, if and only if one filename was specified.")
409        Parser.add_option("-e", "--exceptionfile filename", action="store", type="string", dest="ExceptionFile",
410            help="Specify the name of an exception file, if and only if one filename was specified.")
411        Parser.add_option("-m", "--metadata", action="store_true", type=None, help="Only scan meta-data files information if this option is specified.")
412        Parser.add_option("-s", "--sourcecode", action="store_true", type=None, help="Only scan source code files information if this option is specified.")
413        Parser.add_option("-k", "--keepdatabase", action="store_true", type=None, help="The existing Ecc database will not be cleaned except report information if this option is specified.")
414        Parser.add_option("-l", "--log filename", action="store", dest="LogFile", help="""If specified, the tool should emit the changes that
415                                                                                          were made by the tool after printing the result message.
416                                                                                          If filename, the emit to the file, otherwise emit to
417                                                                                          standard output. If no modifications were made, then do not
418                                                                                          create a log file, or output a log message.""")
419        Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
420        Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
421                                                                                   "including library instances selected, final dependency expression, "\
422                                                                                   "and warning messages, etc.")
423        Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
424        Parser.add_option("-w", "--workspace", action="store", type="string", dest='Workspace', help="Specify workspace.")
425        Parser.add_option("-f", "--folders", action="store_true", type=None, help="Only scanning specified folders which are recorded in config.ini file.")
426
427        (Opt, Args)=Parser.parse_args()
428
429        return (Opt, Args)
430
431##
432#
433# This acts like the main() function for the script, unless it is 'import'ed into another
434# script.
435#
436if __name__ == '__main__':
437    # Initialize log system
438    EdkLogger.Initialize()
439    EdkLogger.IsRaiseError = False
440
441    StartTime = time.clock()
442    Ecc = Ecc()
443    FinishTime = time.clock()
444
445    BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
446    EdkLogger.quiet("\n%s [%s]" % (time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))
447