StrGather.py revision 40d841f6a8f84e75409178e19e69b95e01bada0f 
1# Copyright (c) 2007, Intel Corporation. All rights reserved.<BR>
2# This program and the accompanying materials
3# are licensed and made available under the terms and conditions of the BSD License
4# which accompanies this distribution.  The full text of the license may be found at
5# http://opensource.org/licenses/bsd-license.php
6#
7# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
8# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
9
10#
11#This file is used to parse a strings file and create or add to a string database file.
12#
13
14##
15# Import Modules
16#
17import re
18import Common.EdkLogger as EdkLogger
19from Common.BuildToolError import *
20from UniClassObject import *
21from StringIO import StringIO
22from struct import pack
23
24##
25# Static definitions
26#
27EFI_HII_SIBT_END = '0x00'
28EFI_HII_SIBT_STRING_SCSU = '0x10'
29EFI_HII_SIBT_STRING_SCSU_FONT = '0x11'
30EFI_HII_SIBT_STRINGS_SCSU = '0x12'
31EFI_HII_SIBT_STRINGS_SCSU_FONT = '0x13'
32EFI_HII_SIBT_STRING_UCS2 = '0x14'
33EFI_HII_SIBT_STRING_UCS2_FONT = '0x15'
34EFI_HII_SIBT_STRINGS_UCS2 = '0x16'
35EFI_HII_SIBT_STRINGS_UCS2_FONT = '0x17'
36EFI_HII_SIBT_DUPLICATE = '0x20'
37EFI_HII_SIBT_SKIP2 = '0x21'
38EFI_HII_SIBT_SKIP1 = '0x22'
39EFI_HII_SIBT_EXT1 = '0x30'
40EFI_HII_SIBT_EXT2 = '0x31'
41EFI_HII_SIBT_EXT4 = '0x32'
42EFI_HII_SIBT_FONT = '0x40'
43
44EFI_HII_PACKAGE_STRINGS = '0x04'
45EFI_HII_PACKAGE_FORM = '0x02'
46
47StringPackageType = EFI_HII_PACKAGE_STRINGS
48StringPackageForm = EFI_HII_PACKAGE_FORM
49StringBlockType = EFI_HII_SIBT_STRING_UCS2
50StringSkipType = EFI_HII_SIBT_SKIP2
51
52HexHeader = '0x'
53
54COMMENT = '// '
55DEFINE_STR = '#define'
56COMMENT_DEFINE_STR = COMMENT + DEFINE_STR
57NOT_REFERENCED = 'not referenced'
58COMMENT_NOT_REFERENCED = ' ' + COMMENT + NOT_REFERENCED
59CHAR_ARRAY_DEFIN = 'unsigned char'
60COMMON_FILE_NAME = 'Strings'
61OFFSET = 'offset'
62STRING = 'string'
63TO = 'to'
64STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Z0-9_]+) *\)', re.MULTILINE | re.UNICODE)
65COMPATIBLE_STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Za-z0-9_]+) *\)', re.MULTILINE | re.UNICODE)
66
67EFI_HII_ARRAY_SIZE_LENGTH = 4
68EFI_HII_PACKAGE_HEADER_LENGTH = 4
69EFI_HII_HDR_SIZE_LENGTH = 4
70EFI_HII_STRING_OFFSET_LENGTH = 4
71EFI_STRING_ID = 1
72EFI_STRING_ID_LENGTH = 2
73EFI_HII_LANGUAGE_WINDOW = 0
74EFI_HII_LANGUAGE_WINDOW_LENGTH = 2
75EFI_HII_LANGUAGE_WINDOW_NUMBER = 16
76EFI_HII_STRING_PACKAGE_HDR_LENGTH = EFI_HII_PACKAGE_HEADER_LENGTH + EFI_HII_HDR_SIZE_LENGTH + EFI_HII_STRING_OFFSET_LENGTH + EFI_HII_LANGUAGE_WINDOW_LENGTH * EFI_HII_LANGUAGE_WINDOW_NUMBER + EFI_STRING_ID_LENGTH
77
78H_C_FILE_HEADER = ['//', \
79                   '//  DO NOT EDIT -- auto-generated file', \
80                   '//', \
81                   '//  This file is generated by the StrGather utility', \
82                   '//']
83LANGUAGE_NAME_STRING_NAME = '$LANGUAGE_NAME'
84PRINTABLE_LANGUAGE_NAME_STRING_NAME = '$PRINTABLE_LANGUAGE_NAME'
85
86## Convert a dec number to a hex string
87#
88# Convert a dec number to a formatted hex string in length digit
89# The digit is set to default 8
90# The hex string starts with "0x"
91# DecToHexStr(1000) is '0x000003E8'
92# DecToHexStr(1000, 6) is '0x0003E8'
93#
94# @param Dec:    The number in dec format
95# @param Digit:  The needed digit of hex string
96#
97# @retval:       The formatted hex string
98#
99def DecToHexStr(Dec, Digit = 8):
100    return eval("'0x%0" + str(Digit) + "X' % int(Dec)")
101
102## Convert a dec number to a hex list
103#
104# Convert a dec number to a formatted hex list in size digit
105# The digit is set to default 8
106# DecToHexList(1000) is ['0xE8', '0x03', '0x00', '0x00']
107# DecToHexList(1000, 6) is ['0xE8', '0x03', '0x00']
108#
109# @param Dec:    The number in dec format
110# @param Digit:  The needed digit of hex list
111#
112# @retval:       A list for formatted hex string
113#
114def DecToHexList(Dec, Digit = 8):
115    Hex = eval("'%0" + str(Digit) + "X' % int(Dec)" )
116    List = []
117    for Bit in range(Digit - 2, -1, -2):
118        List.append(HexHeader + Hex[Bit:Bit + 2])
119    return List
120
121## Convert a acsii string to a hex list
122#
123# Convert a acsii string to a formatted hex list
124# AscToHexList('en-US') is ['0x65', '0x6E', '0x2D', '0x55', '0x53']
125#
126# @param Ascii:  The acsii string
127#
128# @retval:       A list for formatted hex string
129#
130def AscToHexList(Ascii):
131    List = []
132    for Item in Ascii:
133        List.append('0x%2X' % ord(Item))
134
135    return List
136
137## Create header of .h file
138#
139# Create a header of .h file
140#
141# @param BaseName: The basename of strings
142#
143# @retval Str:     A string for .h file header
144#
145def CreateHFileHeader(BaseName):
146    Str = ''
147    for Item in H_C_FILE_HEADER:
148        Str = WriteLine(Str, Item)
149    Str = WriteLine(Str, '#ifndef _' + BaseName.upper() + '_STRINGS_DEFINE_H_')
150    Str = WriteLine(Str, '#define _' + BaseName.upper() + '_STRINGS_DEFINE_H_')
151    return Str
152
153## Create content of .h file
154#
155# Create content of .h file
156#
157# @param BaseName:        The basename of strings
158# @param UniObjectClass   A UniObjectClass instance
159# @param IsCompatibleMode Compatible mode
160# @param UniGenCFlag      UniString is generated into AutoGen C file when it is set to True
161#
162# @retval Str:           A string of .h file content
163#
164def CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag):
165    Str = ''
166    ValueStartPtr = 60
167    Line = COMMENT_DEFINE_STR + ' ' + LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(0, 4) + COMMENT_NOT_REFERENCED
168    Str = WriteLine(Str, Line)
169    Line = COMMENT_DEFINE_STR + ' ' + PRINTABLE_LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + PRINTABLE_LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(1, 4) + COMMENT_NOT_REFERENCED
170    Str = WriteLine(Str, Line)
171    for Index in range(2, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]])):
172        StringItem = UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]][Index]
173        Name = StringItem.StringName
174        Token = StringItem.Token
175        Referenced = StringItem.Referenced
176        if Name != None:
177            Line = ''
178            if Referenced == True:
179                if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:
180                    Line = DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4)
181                else:
182                    Line = DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4)
183            else:
184                if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:
185                    Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
186                else:
187                    Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
188            Str = WriteLine(Str, Line)
189
190    Str = WriteLine(Str, '')
191    if IsCompatibleMode or UniGenCFlag:
192        Str = WriteLine(Str, 'extern unsigned char ' + BaseName + 'Strings[];')
193    return Str
194
195## Create a complete .h file
196#
197# Create a complet .h file with file header and file content
198#
199# @param BaseName:        The basename of strings
200# @param UniObjectClass   A UniObjectClass instance
201# @param IsCompatibleMode Compatible mode
202# @param UniGenCFlag      UniString is generated into AutoGen C file when it is set to True
203#
204# @retval Str:           A string of complete .h file
205#
206def CreateHFile(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag):
207    HFile = WriteLine('', CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag))
208
209    return HFile
210
211## Create header of .c file
212#
213# Create a header of .c file
214#
215# @retval Str:     A string for .c file header
216#
217def CreateCFileHeader():
218    Str = ''
219    for Item in H_C_FILE_HEADER:
220        Str = WriteLine(Str, Item)
221
222    return Str
223
224## Create a buffer to store all items in an array
225#
226# @param BinBuffer   Buffer to contain Binary data.
227# @param Array:      The array need to be formatted
228#
229def CreateBinBuffer(BinBuffer, Array):
230    for Item in Array:
231        BinBuffer.write(pack("B", int(Item,16)))
232
233## Create a formatted string all items in an array
234#
235# Use ',' to join each item in an array, and break an new line when reaching the width (default is 16)
236#
237# @param Array:      The array need to be formatted
238# @param Width:      The line length, the default value is set to 16
239#
240# @retval ArrayItem: A string for all formatted array items
241#
242def CreateArrayItem(Array, Width = 16):
243    MaxLength = Width
244    Index = 0
245    Line = '  '
246    ArrayItem = ''
247
248    for Item in Array:
249        if Index < MaxLength:
250            Line = Line + Item + ',  '
251            Index = Index + 1
252        else:
253            ArrayItem = WriteLine(ArrayItem, Line)
254            Line = '  ' + Item +  ',  '
255            Index = 1
256    ArrayItem = Write(ArrayItem, Line.rstrip())
257
258    return ArrayItem
259
260## CreateCFileStringValue
261#
262# Create a line with string value
263#
264# @param Value:  Value of the string
265#
266# @retval Str:   A formatted string with string value
267#
268
269def CreateCFileStringValue(Value):
270    Value = [StringBlockType] + Value
271    Str = WriteLine('', CreateArrayItem(Value))
272
273    return Str
274
275
276## Create content of .c file
277#
278# Create content of .c file
279#
280# @param BaseName:        The basename of strings
281# @param UniObjectClass   A UniObjectClass instance
282# @param IsCompatibleMode Compatible mode
283# @param UniBinBuffer     UniBinBuffer to contain UniBinary data.
284#
285# @retval Str:           A string of .c file content
286#
287def CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniBinBuffer=None):
288    #
289    # Init array length
290    #
291    TotalLength = EFI_HII_ARRAY_SIZE_LENGTH
292    Str = ''
293    Offset = 0
294
295    #
296    # Create lines for each language's strings
297    #
298    for IndexI in range(len(UniObjectClass.LanguageDef)):
299        Language = UniObjectClass.LanguageDef[IndexI][0]
300        LangPrintName = UniObjectClass.LanguageDef[IndexI][1]
301
302        StringBuffer = StringIO()
303        StrStringValue = ''
304        ArrayLength = 0
305        NumberOfUseOtherLangDef = 0
306        Index = 0
307        for IndexJ in range(1, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[IndexI][0]])):
308            Item = UniObjectClass.FindByToken(IndexJ, Language)
309            Name = Item.StringName
310            Value = Item.StringValueByteList
311            Referenced = Item.Referenced
312            Token = Item.Token
313            Length = Item.Length
314            UseOtherLangDef = Item.UseOtherLangDef
315
316            if UseOtherLangDef != '' and Referenced:
317                NumberOfUseOtherLangDef = NumberOfUseOtherLangDef + 1
318                Index = Index + 1
319            else:
320                if NumberOfUseOtherLangDef > 0:
321                    StrStringValue = WriteLine(StrStringValue, CreateArrayItem([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4)))
322                    CreateBinBuffer (StringBuffer, ([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4)))
323                    NumberOfUseOtherLangDef = 0
324                    ArrayLength = ArrayLength + 3
325                if Referenced and Item.Token > 0:
326                    Index = Index + 1
327                    StrStringValue = WriteLine(StrStringValue, "// %s: %s:%s" % (DecToHexStr(Index, 4), Name, DecToHexStr(Token, 4)))
328                    StrStringValue = Write(StrStringValue, CreateCFileStringValue(Value))
329                    CreateBinBuffer (StringBuffer, [StringBlockType] + Value)
330                    ArrayLength = ArrayLength + Item.Length + 1 # 1 is for the length of string type
331
332        #
333        # EFI_HII_PACKAGE_HEADER
334        #
335        Offset = EFI_HII_STRING_PACKAGE_HDR_LENGTH + len(Language) + 1
336        ArrayLength = Offset + ArrayLength + 1
337
338        #
339        # Create PACKAGE HEADER
340        #
341        Str = WriteLine(Str, '// PACKAGE HEADER\n')
342        TotalLength = TotalLength + ArrayLength
343
344        List = DecToHexList(ArrayLength, 6) + \
345               [StringPackageType] + \
346               DecToHexList(Offset) + \
347               DecToHexList(Offset) + \
348               DecToHexList(EFI_HII_LANGUAGE_WINDOW, EFI_HII_LANGUAGE_WINDOW_LENGTH * 2) * EFI_HII_LANGUAGE_WINDOW_NUMBER + \
349               DecToHexList(EFI_STRING_ID, 4) + \
350               AscToHexList(Language) + \
351               DecToHexList(0, 2)
352        Str = WriteLine(Str, CreateArrayItem(List, 16) + '\n')
353
354        #
355        # Create PACKAGE DATA
356        #
357        Str = WriteLine(Str, '// PACKAGE DATA\n')
358        Str = Write(Str, StrStringValue)
359
360        #
361        # Add an EFI_HII_SIBT_END at last
362        #
363        Str = WriteLine(Str, '  ' + EFI_HII_SIBT_END + ",")
364
365        #
366        # Create binary UNI string
367        #
368        if UniBinBuffer:
369            CreateBinBuffer (UniBinBuffer, List)
370            UniBinBuffer.write (StringBuffer.getvalue())
371            UniBinBuffer.write (pack("B", int(EFI_HII_SIBT_END,16)))
372        StringBuffer.close()
373
374    #
375    # Create line for string variable name
376    # "unsigned char $(BaseName)Strings[] = {"
377    #
378    AllStr = WriteLine('', CHAR_ARRAY_DEFIN + ' ' + BaseName + COMMON_FILE_NAME + '[] = {\n' )
379
380    if IsCompatibleMode:
381        #
382        # Create FRAMEWORK_EFI_HII_PACK_HEADER in compatible mode
383        #
384        AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Length')
385        AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength + 2)) + '\n')
386        AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Type')
387        AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(2, 4)) + '\n')
388    else:
389        #
390        # Create whole array length in UEFI mode
391        #
392        AllStr = WriteLine(AllStr, '// STRGATHER_OUTPUT_HEADER')
393        AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength)) + '\n')
394
395    #
396    # Join package data
397    #
398    AllStr = Write(AllStr, Str)
399
400    return AllStr
401
402## Create end of .c file
403#
404# Create end of .c file
405#
406# @retval Str:           A string of .h file end
407#
408def CreateCFileEnd():
409    Str = Write('', '};')
410    return Str
411
412## Create a .c file
413#
414# Create a complete .c file
415#
416# @param BaseName:        The basename of strings
417# @param UniObjectClass   A UniObjectClass instance
418# @param IsCompatibleMode Compatible Mode
419#
420# @retval CFile:         A string of complete .c file
421#
422def CreateCFile(BaseName, UniObjectClass, IsCompatibleMode):
423    CFile = ''
424    #CFile = WriteLine(CFile, CreateCFileHeader())
425    CFile = WriteLine(CFile, CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode))
426    CFile = WriteLine(CFile, CreateCFileEnd())
427    return CFile
428
429## GetFileList
430#
431# Get a list for all files
432#
433# @param IncludeList:  A list of all path to be searched
434# @param SkipList:     A list of all types of file could be skipped
435#
436# @retval FileList:    A list of all files found
437#
438def GetFileList(SourceFileList, IncludeList, SkipList):
439    if IncludeList == None:
440        EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, "Include path for unicode file is not defined")
441
442    FileList = []
443    if SkipList == None:
444        SkipList = []
445
446    for File in SourceFileList:
447        for Dir in IncludeList:
448            if not os.path.exists(Dir):
449                continue
450            File = os.path.join(Dir, File.Path)
451            #
452            # Ignore Dir
453            #
454            if os.path.isfile(File) != True:
455                continue
456            #
457            # Ignore file listed in skip list
458            #
459            IsSkip = False
460            for Skip in SkipList:
461                if os.path.splitext(File)[1].upper() == Skip.upper():
462                    EdkLogger.verbose("Skipped %s for string token uses search" % File)
463                    IsSkip = True
464                    break
465
466            if not IsSkip:
467                FileList.append(File)
468
469            break
470
471    return FileList
472
473## SearchString
474#
475# Search whether all string defined in UniObjectClass are referenced
476# All string used should be set to Referenced
477#
478# @param UniObjectClass:  Input UniObjectClass
479# @param FileList:        Search path list
480# @param IsCompatibleMode Compatible Mode
481#
482# @retval UniObjectClass: UniObjectClass after searched
483#
484def SearchString(UniObjectClass, FileList, IsCompatibleMode):
485    if FileList == []:
486        return UniObjectClass
487
488    for File in FileList:
489        if os.path.isfile(File):
490            Lines = open(File, 'r')
491            for Line in Lines:
492                if not IsCompatibleMode:
493                    StringTokenList = STRING_TOKEN.findall(Line)
494                else:
495                    StringTokenList = COMPATIBLE_STRING_TOKEN.findall(Line)
496                for StrName in StringTokenList:
497                    EdkLogger.debug(EdkLogger.DEBUG_5, "Found string identifier: " + StrName)
498                    UniObjectClass.SetStringReferenced(StrName)
499
500    UniObjectClass.ReToken()
501
502    return UniObjectClass
503
504## GetStringFiles
505#
506# This function is used for UEFI2.1 spec
507#
508#
509def GetStringFiles(UniFilList, SourceFileList, IncludeList, SkipList, BaseName, IsCompatibleMode = False, ShellMode = False, UniGenCFlag = True, UniGenBinBuffer = None):
510    Status = True
511    ErrorMessage = ''
512
513    if len(UniFilList) > 0:
514        if ShellMode:
515            #
516            # support ISO 639-2 codes in .UNI files of EDK Shell
517            #
518            Uni = UniFileClassObject(UniFilList, True)
519        else:
520            Uni = UniFileClassObject(UniFilList, IsCompatibleMode)
521    else:
522        EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, 'No unicode files given')
523
524    FileList = GetFileList(SourceFileList, IncludeList, SkipList)
525
526    Uni = SearchString(Uni, FileList, IsCompatibleMode)
527
528    HFile = CreateHFile(BaseName, Uni, IsCompatibleMode, UniGenCFlag)
529    CFile = None
530    if IsCompatibleMode or UniGenCFlag:
531        CFile = CreateCFile(BaseName, Uni, IsCompatibleMode)
532    if UniGenBinBuffer:
533        CreateCFileContent(BaseName, Uni, IsCompatibleMode, UniGenBinBuffer)
534
535    return HFile, CFile
536
537#
538# Write an item
539#
540def Write(Target, Item):
541    return Target + Item
542
543#
544# Write an item with a break line
545#
546def WriteLine(Target, Item):
547    return Target + Item + '\n'
548
549# This acts like the main() function for the script, unless it is 'import'ed into another
550# script.
551if __name__ == '__main__':
552    EdkLogger.info('start')
553
554    UniFileList = [
555                   r'C:\\Edk\\Strings2.uni',
556                   r'C:\\Edk\\Strings.uni'
557    ]
558
559    SrcFileList = []
560    for Root, Dirs, Files in os.walk('C:\\Edk'):
561        for File in Files:
562            SrcFileList.append(File)
563
564    IncludeList = [
565                   r'C:\\Edk'
566    ]
567
568    SkipList = ['.inf', '.uni']
569    BaseName = 'DriverSample'
570    (h, c) = GetStringFiles(UniFileList, SrcFileList, IncludeList, SkipList, BaseName, True)
571    hfile = open('unistring.h', 'w')
572    cfile = open('unistring.c', 'w')
573    hfile.write(h)
574    cfile.write(c)
575
576    EdkLogger.info('end')
577