StrGather.py revision 756ad8f8e9d3e345f1883546e2a3125203e234aa 
1# Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
2# This program and the accompanying materials
3# are licensed and made available under the terms and conditions of the BSD License
4# which accompanies this distribution.  The full text of the license may be found at
5# http://opensource.org/licenses/bsd-license.php
6#
7# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
8# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
9
10#
11#This file is used to parse a strings file and create or add to a string database file.
12#
13
14##
15# Import Modules
16#
17import re
18import Common.EdkLogger as EdkLogger
19from Common.BuildToolError import *
20from UniClassObject import *
21from StringIO import StringIO
22from struct import pack
23
24##
25# Static definitions
26#
27EFI_HII_SIBT_END = '0x00'
28EFI_HII_SIBT_STRING_SCSU = '0x10'
29EFI_HII_SIBT_STRING_SCSU_FONT = '0x11'
30EFI_HII_SIBT_STRINGS_SCSU = '0x12'
31EFI_HII_SIBT_STRINGS_SCSU_FONT = '0x13'
32EFI_HII_SIBT_STRING_UCS2 = '0x14'
33EFI_HII_SIBT_STRING_UCS2_FONT = '0x15'
34EFI_HII_SIBT_STRINGS_UCS2 = '0x16'
35EFI_HII_SIBT_STRINGS_UCS2_FONT = '0x17'
36EFI_HII_SIBT_DUPLICATE = '0x20'
37EFI_HII_SIBT_SKIP2 = '0x21'
38EFI_HII_SIBT_SKIP1 = '0x22'
39EFI_HII_SIBT_EXT1 = '0x30'
40EFI_HII_SIBT_EXT2 = '0x31'
41EFI_HII_SIBT_EXT4 = '0x32'
42EFI_HII_SIBT_FONT = '0x40'
43
44EFI_HII_PACKAGE_STRINGS = '0x04'
45EFI_HII_PACKAGE_FORM = '0x02'
46
47StringPackageType = EFI_HII_PACKAGE_STRINGS
48StringPackageForm = EFI_HII_PACKAGE_FORM
49StringBlockType = EFI_HII_SIBT_STRING_UCS2
50StringSkipType = EFI_HII_SIBT_SKIP2
51
52HexHeader = '0x'
53
54COMMENT = '// '
55DEFINE_STR = '#define'
56COMMENT_DEFINE_STR = COMMENT + DEFINE_STR
57NOT_REFERENCED = 'not referenced'
58COMMENT_NOT_REFERENCED = ' ' + COMMENT + NOT_REFERENCED
59CHAR_ARRAY_DEFIN = 'unsigned char'
60COMMON_FILE_NAME = 'Strings'
61OFFSET = 'offset'
62STRING = 'string'
63TO = 'to'
64STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Z0-9_]+) *\)', re.MULTILINE | re.UNICODE)
65COMPATIBLE_STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Za-z0-9_]+) *\)', re.MULTILINE | re.UNICODE)
66
67EFI_HII_ARRAY_SIZE_LENGTH = 4
68EFI_HII_PACKAGE_HEADER_LENGTH = 4
69EFI_HII_HDR_SIZE_LENGTH = 4
70EFI_HII_STRING_OFFSET_LENGTH = 4
71EFI_STRING_ID = 1
72EFI_STRING_ID_LENGTH = 2
73EFI_HII_LANGUAGE_WINDOW = 0
74EFI_HII_LANGUAGE_WINDOW_LENGTH = 2
75EFI_HII_LANGUAGE_WINDOW_NUMBER = 16
76EFI_HII_STRING_PACKAGE_HDR_LENGTH = EFI_HII_PACKAGE_HEADER_LENGTH + EFI_HII_HDR_SIZE_LENGTH + EFI_HII_STRING_OFFSET_LENGTH + EFI_HII_LANGUAGE_WINDOW_LENGTH * EFI_HII_LANGUAGE_WINDOW_NUMBER + EFI_STRING_ID_LENGTH
77
78H_C_FILE_HEADER = ['//', \
79                   '//  DO NOT EDIT -- auto-generated file', \
80                   '//', \
81                   '//  This file is generated by the StrGather utility', \
82                   '//']
83LANGUAGE_NAME_STRING_NAME = '$LANGUAGE_NAME'
84PRINTABLE_LANGUAGE_NAME_STRING_NAME = '$PRINTABLE_LANGUAGE_NAME'
85
86## Convert a dec number to a hex string
87#
88# Convert a dec number to a formatted hex string in length digit
89# The digit is set to default 8
90# The hex string starts with "0x"
91# DecToHexStr(1000) is '0x000003E8'
92# DecToHexStr(1000, 6) is '0x0003E8'
93#
94# @param Dec:    The number in dec format
95# @param Digit:  The needed digit of hex string
96#
97# @retval:       The formatted hex string
98#
99def DecToHexStr(Dec, Digit = 8):
100    return eval("'0x%0" + str(Digit) + "X' % int(Dec)")
101
102## Convert a dec number to a hex list
103#
104# Convert a dec number to a formatted hex list in size digit
105# The digit is set to default 8
106# DecToHexList(1000) is ['0xE8', '0x03', '0x00', '0x00']
107# DecToHexList(1000, 6) is ['0xE8', '0x03', '0x00']
108#
109# @param Dec:    The number in dec format
110# @param Digit:  The needed digit of hex list
111#
112# @retval:       A list for formatted hex string
113#
114def DecToHexList(Dec, Digit = 8):
115    Hex = eval("'%0" + str(Digit) + "X' % int(Dec)" )
116    List = []
117    for Bit in range(Digit - 2, -1, -2):
118        List.append(HexHeader + Hex[Bit:Bit + 2])
119    return List
120
121## Convert a acsii string to a hex list
122#
123# Convert a acsii string to a formatted hex list
124# AscToHexList('en-US') is ['0x65', '0x6E', '0x2D', '0x55', '0x53']
125#
126# @param Ascii:  The acsii string
127#
128# @retval:       A list for formatted hex string
129#
130def AscToHexList(Ascii):
131    List = []
132    for Item in Ascii:
133        List.append('0x%2X' % ord(Item))
134
135    return List
136
137## Create header of .h file
138#
139# Create a header of .h file
140#
141# @param BaseName: The basename of strings
142#
143# @retval Str:     A string for .h file header
144#
145def CreateHFileHeader(BaseName):
146    Str = ''
147    for Item in H_C_FILE_HEADER:
148        Str = WriteLine(Str, Item)
149    Str = WriteLine(Str, '#ifndef _' + BaseName.upper() + '_STRINGS_DEFINE_H_')
150    Str = WriteLine(Str, '#define _' + BaseName.upper() + '_STRINGS_DEFINE_H_')
151    return Str
152
153## Create content of .h file
154#
155# Create content of .h file
156#
157# @param BaseName:        The basename of strings
158# @param UniObjectClass   A UniObjectClass instance
159# @param IsCompatibleMode Compatible mode
160# @param UniGenCFlag      UniString is generated into AutoGen C file when it is set to True
161#
162# @retval Str:           A string of .h file content
163#
164def CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag):
165    Str = ''
166    ValueStartPtr = 60
167    Line = COMMENT_DEFINE_STR + ' ' + LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(0, 4) + COMMENT_NOT_REFERENCED
168    Str = WriteLine(Str, Line)
169    Line = COMMENT_DEFINE_STR + ' ' + PRINTABLE_LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + PRINTABLE_LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(1, 4) + COMMENT_NOT_REFERENCED
170    Str = WriteLine(Str, Line)
171
172    #Group the referred STRING token together.
173    for Index in range(2, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]])):
174        StringItem = UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]][Index]
175        Name = StringItem.StringName
176        Token = StringItem.Token
177        Referenced = StringItem.Referenced
178        if Name != None:
179            Line = ''
180            if Referenced == True:
181                if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:
182                    Line = DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4)
183                else:
184                    Line = DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4)
185                Str = WriteLine(Str, Line)
186
187    #Group the unused STRING token together.
188    for Index in range(2, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]])):
189        StringItem = UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]][Index]
190        Name = StringItem.StringName
191        Token = StringItem.Token
192        Referenced = StringItem.Referenced
193        if Name != None:
194            Line = ''
195            if Referenced == False:
196                if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:
197                    Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
198                else:
199                    Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
200                Str = WriteLine(Str, Line)
201
202    Str = WriteLine(Str, '')
203    if IsCompatibleMode or UniGenCFlag:
204        Str = WriteLine(Str, 'extern unsigned char ' + BaseName + 'Strings[];')
205    return Str
206
207## Create a complete .h file
208#
209# Create a complet .h file with file header and file content
210#
211# @param BaseName:        The basename of strings
212# @param UniObjectClass   A UniObjectClass instance
213# @param IsCompatibleMode Compatible mode
214# @param UniGenCFlag      UniString is generated into AutoGen C file when it is set to True
215#
216# @retval Str:           A string of complete .h file
217#
218def CreateHFile(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag):
219    HFile = WriteLine('', CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag))
220
221    return HFile
222
223## Create header of .c file
224#
225# Create a header of .c file
226#
227# @retval Str:     A string for .c file header
228#
229def CreateCFileHeader():
230    Str = ''
231    for Item in H_C_FILE_HEADER:
232        Str = WriteLine(Str, Item)
233
234    return Str
235
236## Create a buffer to store all items in an array
237#
238# @param BinBuffer   Buffer to contain Binary data.
239# @param Array:      The array need to be formatted
240#
241def CreateBinBuffer(BinBuffer, Array):
242    for Item in Array:
243        BinBuffer.write(pack("B", int(Item,16)))
244
245## Create a formatted string all items in an array
246#
247# Use ',' to join each item in an array, and break an new line when reaching the width (default is 16)
248#
249# @param Array:      The array need to be formatted
250# @param Width:      The line length, the default value is set to 16
251#
252# @retval ArrayItem: A string for all formatted array items
253#
254def CreateArrayItem(Array, Width = 16):
255    MaxLength = Width
256    Index = 0
257    Line = '  '
258    ArrayItem = ''
259
260    for Item in Array:
261        if Index < MaxLength:
262            Line = Line + Item + ',  '
263            Index = Index + 1
264        else:
265            ArrayItem = WriteLine(ArrayItem, Line)
266            Line = '  ' + Item +  ',  '
267            Index = 1
268    ArrayItem = Write(ArrayItem, Line.rstrip())
269
270    return ArrayItem
271
272## CreateCFileStringValue
273#
274# Create a line with string value
275#
276# @param Value:  Value of the string
277#
278# @retval Str:   A formatted string with string value
279#
280
281def CreateCFileStringValue(Value):
282    Value = [StringBlockType] + Value
283    Str = WriteLine('', CreateArrayItem(Value))
284
285    return Str
286
287
288## Create content of .c file
289#
290# Create content of .c file
291#
292# @param BaseName:        The basename of strings
293# @param UniObjectClass   A UniObjectClass instance
294# @param IsCompatibleMode Compatible mode
295# @param UniBinBuffer     UniBinBuffer to contain UniBinary data.
296#
297# @retval Str:           A string of .c file content
298#
299def CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniBinBuffer=None):
300    #
301    # Init array length
302    #
303    TotalLength = EFI_HII_ARRAY_SIZE_LENGTH
304    Str = ''
305    Offset = 0
306
307    #
308    # Create lines for each language's strings
309    #
310    for IndexI in range(len(UniObjectClass.LanguageDef)):
311        Language = UniObjectClass.LanguageDef[IndexI][0]
312        LangPrintName = UniObjectClass.LanguageDef[IndexI][1]
313
314        StringBuffer = StringIO()
315        StrStringValue = ''
316        ArrayLength = 0
317        NumberOfUseOtherLangDef = 0
318        Index = 0
319        for IndexJ in range(1, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[IndexI][0]])):
320            Item = UniObjectClass.FindByToken(IndexJ, Language)
321            Name = Item.StringName
322            Value = Item.StringValueByteList
323            Referenced = Item.Referenced
324            Token = Item.Token
325            Length = Item.Length
326            UseOtherLangDef = Item.UseOtherLangDef
327
328            if UseOtherLangDef != '' and Referenced:
329                NumberOfUseOtherLangDef = NumberOfUseOtherLangDef + 1
330                Index = Index + 1
331            else:
332                if NumberOfUseOtherLangDef > 0:
333                    StrStringValue = WriteLine(StrStringValue, CreateArrayItem([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4)))
334                    CreateBinBuffer (StringBuffer, ([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4)))
335                    NumberOfUseOtherLangDef = 0
336                    ArrayLength = ArrayLength + 3
337                if Referenced and Item.Token > 0:
338                    Index = Index + 1
339                    StrStringValue = WriteLine(StrStringValue, "// %s: %s:%s" % (DecToHexStr(Index, 4), Name, DecToHexStr(Token, 4)))
340                    StrStringValue = Write(StrStringValue, CreateCFileStringValue(Value))
341                    CreateBinBuffer (StringBuffer, [StringBlockType] + Value)
342                    ArrayLength = ArrayLength + Item.Length + 1 # 1 is for the length of string type
343
344        #
345        # EFI_HII_PACKAGE_HEADER
346        #
347        Offset = EFI_HII_STRING_PACKAGE_HDR_LENGTH + len(Language) + 1
348        ArrayLength = Offset + ArrayLength + 1
349
350        #
351        # Create PACKAGE HEADER
352        #
353        Str = WriteLine(Str, '// PACKAGE HEADER\n')
354        TotalLength = TotalLength + ArrayLength
355
356        List = DecToHexList(ArrayLength, 6) + \
357               [StringPackageType] + \
358               DecToHexList(Offset) + \
359               DecToHexList(Offset) + \
360               DecToHexList(EFI_HII_LANGUAGE_WINDOW, EFI_HII_LANGUAGE_WINDOW_LENGTH * 2) * EFI_HII_LANGUAGE_WINDOW_NUMBER + \
361               DecToHexList(EFI_STRING_ID, 4) + \
362               AscToHexList(Language) + \
363               DecToHexList(0, 2)
364        Str = WriteLine(Str, CreateArrayItem(List, 16) + '\n')
365
366        #
367        # Create PACKAGE DATA
368        #
369        Str = WriteLine(Str, '// PACKAGE DATA\n')
370        Str = Write(Str, StrStringValue)
371
372        #
373        # Add an EFI_HII_SIBT_END at last
374        #
375        Str = WriteLine(Str, '  ' + EFI_HII_SIBT_END + ",")
376
377        #
378        # Create binary UNI string
379        #
380        if UniBinBuffer:
381            CreateBinBuffer (UniBinBuffer, List)
382            UniBinBuffer.write (StringBuffer.getvalue())
383            UniBinBuffer.write (pack("B", int(EFI_HII_SIBT_END,16)))
384        StringBuffer.close()
385
386    #
387    # Create line for string variable name
388    # "unsigned char $(BaseName)Strings[] = {"
389    #
390    AllStr = WriteLine('', CHAR_ARRAY_DEFIN + ' ' + BaseName + COMMON_FILE_NAME + '[] = {\n' )
391
392    if IsCompatibleMode:
393        #
394        # Create FRAMEWORK_EFI_HII_PACK_HEADER in compatible mode
395        #
396        AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Length')
397        AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength + 2)) + '\n')
398        AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Type')
399        AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(2, 4)) + '\n')
400    else:
401        #
402        # Create whole array length in UEFI mode
403        #
404        AllStr = WriteLine(AllStr, '// STRGATHER_OUTPUT_HEADER')
405        AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength)) + '\n')
406
407    #
408    # Join package data
409    #
410    AllStr = Write(AllStr, Str)
411
412    return AllStr
413
414## Create end of .c file
415#
416# Create end of .c file
417#
418# @retval Str:           A string of .h file end
419#
420def CreateCFileEnd():
421    Str = Write('', '};')
422    return Str
423
424## Create a .c file
425#
426# Create a complete .c file
427#
428# @param BaseName:        The basename of strings
429# @param UniObjectClass   A UniObjectClass instance
430# @param IsCompatibleMode Compatible Mode
431#
432# @retval CFile:         A string of complete .c file
433#
434def CreateCFile(BaseName, UniObjectClass, IsCompatibleMode):
435    CFile = ''
436    #CFile = WriteLine(CFile, CreateCFileHeader())
437    CFile = WriteLine(CFile, CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode))
438    CFile = WriteLine(CFile, CreateCFileEnd())
439    return CFile
440
441## GetFileList
442#
443# Get a list for all files
444#
445# @param IncludeList:  A list of all path to be searched
446# @param SkipList:     A list of all types of file could be skipped
447#
448# @retval FileList:    A list of all files found
449#
450def GetFileList(SourceFileList, IncludeList, SkipList):
451    if IncludeList == None:
452        EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, "Include path for unicode file is not defined")
453
454    FileList = []
455    if SkipList == None:
456        SkipList = []
457
458    for File in SourceFileList:
459        for Dir in IncludeList:
460            if not os.path.exists(Dir):
461                continue
462            File = os.path.join(Dir, File.Path)
463            #
464            # Ignore Dir
465            #
466            if os.path.isfile(File) != True:
467                continue
468            #
469            # Ignore file listed in skip list
470            #
471            IsSkip = False
472            for Skip in SkipList:
473                if os.path.splitext(File)[1].upper() == Skip.upper():
474                    EdkLogger.verbose("Skipped %s for string token uses search" % File)
475                    IsSkip = True
476                    break
477
478            if not IsSkip:
479                FileList.append(File)
480
481            break
482
483    return FileList
484
485## SearchString
486#
487# Search whether all string defined in UniObjectClass are referenced
488# All string used should be set to Referenced
489#
490# @param UniObjectClass:  Input UniObjectClass
491# @param FileList:        Search path list
492# @param IsCompatibleMode Compatible Mode
493#
494# @retval UniObjectClass: UniObjectClass after searched
495#
496def SearchString(UniObjectClass, FileList, IsCompatibleMode):
497    if FileList == []:
498        return UniObjectClass
499
500    for File in FileList:
501        if os.path.isfile(File):
502            Lines = open(File, 'r')
503            for Line in Lines:
504                if not IsCompatibleMode:
505                    StringTokenList = STRING_TOKEN.findall(Line)
506                else:
507                    StringTokenList = COMPATIBLE_STRING_TOKEN.findall(Line)
508                for StrName in StringTokenList:
509                    EdkLogger.debug(EdkLogger.DEBUG_5, "Found string identifier: " + StrName)
510                    UniObjectClass.SetStringReferenced(StrName)
511
512    UniObjectClass.ReToken()
513
514    return UniObjectClass
515
516## GetStringFiles
517#
518# This function is used for UEFI2.1 spec
519#
520#
521def GetStringFiles(UniFilList, SourceFileList, IncludeList, IncludePathList, SkipList, BaseName, IsCompatibleMode = False, ShellMode = False, UniGenCFlag = True, UniGenBinBuffer = None):
522    Status = True
523    ErrorMessage = ''
524
525    if len(UniFilList) > 0:
526        if ShellMode:
527            #
528            # support ISO 639-2 codes in .UNI files of EDK Shell
529            #
530            Uni = UniFileClassObject(sorted (UniFilList), True, IncludePathList)
531        else:
532            Uni = UniFileClassObject(sorted (UniFilList), IsCompatibleMode, IncludePathList)
533    else:
534        EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, 'No unicode files given')
535
536    FileList = GetFileList(SourceFileList, IncludeList, SkipList)
537
538    Uni = SearchString(Uni, sorted (FileList), IsCompatibleMode)
539
540    HFile = CreateHFile(BaseName, Uni, IsCompatibleMode, UniGenCFlag)
541    CFile = None
542    if IsCompatibleMode or UniGenCFlag:
543        CFile = CreateCFile(BaseName, Uni, IsCompatibleMode)
544    if UniGenBinBuffer:
545        CreateCFileContent(BaseName, Uni, IsCompatibleMode, UniGenBinBuffer)
546
547    return HFile, CFile
548
549#
550# Write an item
551#
552def Write(Target, Item):
553    return Target + Item
554
555#
556# Write an item with a break line
557#
558def WriteLine(Target, Item):
559    return Target + Item + '\n'
560
561# This acts like the main() function for the script, unless it is 'import'ed into another
562# script.
563if __name__ == '__main__':
564    EdkLogger.info('start')
565
566    UniFileList = [
567                   r'C:\\Edk\\Strings2.uni',
568                   r'C:\\Edk\\Strings.uni'
569    ]
570
571    SrcFileList = []
572    for Root, Dirs, Files in os.walk('C:\\Edk'):
573        for File in Files:
574            SrcFileList.append(File)
575
576    IncludeList = [
577                   r'C:\\Edk'
578    ]
579
580    SkipList = ['.inf', '.uni']
581    BaseName = 'DriverSample'
582    (h, c) = GetStringFiles(UniFileList, SrcFileList, IncludeList, SkipList, BaseName, True)
583    hfile = open('unistring.h', 'w')
584    cfile = open('unistring.c', 'w')
585    hfile.write(h)
586    cfile.write(c)
587
588    EdkLogger.info('end')
589