1/*
2*******************************************************************************
3*
4*   Copyright (C) 2003-2012, International Business Machines
5*   Corporation and others.  All Rights Reserved.
6*
7*******************************************************************************
8*   file name:  gensprep.c
9*   encoding:   US-ASCII
10*   tab size:   8 (not used)
11*   indentation:4
12*
13*   created on: 2003-02-06
14*   created by: Ram Viswanadha
15*
16*   This program reads the Profile.txt files,
17*   parses them, and extracts the data for StringPrep profile.
18*   It then preprocesses it and writes a binary file for efficient use
19*   in various StringPrep conversion processes.
20*/
21
22#define USPREP_TYPE_NAMES_ARRAY 1
23
24#include <stdio.h>
25#include <stdlib.h>
26
27#include "cmemory.h"
28#include "cstring.h"
29#include "unewdata.h"
30#include "uoptions.h"
31#include "uparse.h"
32#include "sprpimpl.h"
33
34#include "unicode/uclean.h"
35#include "unicode/udata.h"
36#include "unicode/utypes.h"
37#include "unicode/putil.h"
38
39
40U_CDECL_BEGIN
41#include "gensprep.h"
42U_CDECL_END
43
44UBool beVerbose=FALSE, haveCopyright=TRUE;
45
46#define NORM_CORRECTIONS_FILE_NAME "NormalizationCorrections.txt"
47
48#define NORMALIZE_DIRECTIVE "normalize"
49#define NORMALIZE_DIRECTIVE_LEN 9
50#define CHECK_BIDI_DIRECTIVE "check-bidi"
51#define CHECK_BIDI_DIRECTIVE_LEN 10
52
53/* prototypes --------------------------------------------------------------- */
54
55static void
56parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode);
57
58static void
59parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode);
60
61
62/* -------------------------------------------------------------------------- */
63
64static UOption options[]={
65    UOPTION_HELP_H,
66    UOPTION_HELP_QUESTION_MARK,
67    UOPTION_VERBOSE,
68    UOPTION_COPYRIGHT,
69    UOPTION_DESTDIR,
70    UOPTION_SOURCEDIR,
71    UOPTION_ICUDATADIR,
72    UOPTION_BUNDLE_NAME,
73    { "normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0 },
74    { "norm-correction", NULL, NULL, NULL, 'm', UOPT_REQUIRES_ARG, 0 },
75    { "check-bidi", NULL, NULL, NULL,  'k', UOPT_NO_ARG, 0},
76    { "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 },
77};
78
79enum{
80    HELP,
81    HELP_QUESTION_MARK,
82    VERBOSE,
83    COPYRIGHT,
84    DESTDIR,
85    SOURCEDIR,
86    ICUDATADIR,
87    BUNDLE_NAME,
88    NORMALIZE,
89    NORM_CORRECTION_DIR,
90    CHECK_BIDI,
91    UNICODE_VERSION
92};
93
94static int printHelp(int argc, char* argv[]){
95    /*
96     * Broken into chucks because the C89 standard says the minimum
97     * required supported string length is 509 bytes.
98     */
99    fprintf(stderr,
100        "Usage: %s [-options] [file_name]\n"
101        "\n"
102        "Read the files specified and\n"
103        "create a binary file [package-name]_[bundle-name]." DATA_TYPE " with the StringPrep profile data\n"
104        "\n",
105        argv[0]);
106    fprintf(stderr,
107        "Options:\n"
108        "\t-h or -? or --help       print this usage text\n"
109        "\t-v or --verbose          verbose output\n"
110        "\t-c or --copyright        include a copyright notice\n");
111    fprintf(stderr,
112        "\t-d or --destdir          destination directory, followed by the path\n"
113        "\t-s or --sourcedir        source directory of ICU data, followed by the path\n"
114        "\t-b or --bundle-name      generate the ouput data file with the name specified\n"
115        "\t-i or --icudatadir       directory for locating any needed intermediate data files,\n"
116        "\t                         followed by path, defaults to %s\n",
117        u_getDataDirectory());
118    fprintf(stderr,
119        "\t-n or --normalize        turn on the option for normalization and include mappings\n"
120        "\t                         from NormalizationCorrections.txt from the given path,\n"
121        "\t                         e.g: /test/icu/source/data/unidata\n");
122    fprintf(stderr,
123        "\t-m or --norm-correction  use NormalizationCorrections.txt from the given path\n"
124        "\t                         when the input file contains a normalization directive.\n"
125        "\t                         unlike -n/--normalize, this option does not force the\n"
126        "\t                         normalization.\n");
127    fprintf(stderr,
128        "\t-k or --check-bidi       turn on the option for checking for BiDi in the profile\n"
129        "\t-u or --unicode          version of Unicode to be used with this profile followed by the version\n"
130        );
131    return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
132}
133
134
135extern int
136main(int argc, char* argv[]) {
137#if !UCONFIG_NO_IDNA
138    char* filename = NULL;
139#endif
140    const char *srcDir=NULL, *destDir=NULL, *icuUniDataDir=NULL;
141    const char *bundleName=NULL, *inputFileName = NULL;
142    char *basename=NULL;
143    int32_t sprepOptions = 0;
144
145    UErrorCode errorCode=U_ZERO_ERROR;
146
147    U_MAIN_INIT_ARGS(argc, argv);
148
149    /* preset then read command line options */
150    options[DESTDIR].value=u_getDataDirectory();
151    options[SOURCEDIR].value="";
152    options[UNICODE_VERSION].value="0"; /* don't assume the unicode version */
153    options[BUNDLE_NAME].value = DATA_NAME;
154    options[NORMALIZE].value = "";
155
156    argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
157
158    /* error handling, printing usage message */
159    if(argc<0) {
160        fprintf(stderr,
161            "error in command line argument \"%s\"\n",
162            argv[-argc]);
163    }
164    if(argc<0 || options[HELP].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
165        return printHelp(argc, argv);
166
167    }
168
169    /* get the options values */
170    beVerbose=options[VERBOSE].doesOccur;
171    haveCopyright=options[COPYRIGHT].doesOccur;
172    srcDir=options[SOURCEDIR].value;
173    destDir=options[DESTDIR].value;
174    bundleName = options[BUNDLE_NAME].value;
175    if(options[NORMALIZE].doesOccur) {
176        icuUniDataDir = options[NORMALIZE].value;
177    } else {
178        icuUniDataDir = options[NORM_CORRECTION_DIR].value;
179    }
180
181    if(argc<2) {
182        /* print the help message */
183        return printHelp(argc, argv);
184    } else {
185        inputFileName = argv[1];
186    }
187    if(!options[UNICODE_VERSION].doesOccur){
188        return printHelp(argc, argv);
189    }
190    if(options[ICUDATADIR].doesOccur) {
191        u_setDataDirectory(options[ICUDATADIR].value);
192    }
193#if UCONFIG_NO_IDNA
194
195    fprintf(stderr,
196        "gensprep writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE
197        " because UCONFIG_NO_IDNA is set, \n"
198        "see icu/source/common/unicode/uconfig.h\n");
199    generateData(destDir, bundleName);
200
201#else
202
203    setUnicodeVersion(options[UNICODE_VERSION].value);
204    filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + 300); /* hopefully this should be enough */
205
206    /* prepare the filename beginning with the source dir */
207    if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL && uprv_strchr(srcDir,U_FILE_ALT_SEP_CHAR) == NULL){
208        filename[0] = '.';
209        filename[1] = U_FILE_SEP_CHAR;
210        uprv_strcpy(filename+2,srcDir);
211    }else{
212        uprv_strcpy(filename, srcDir);
213    }
214
215    basename=filename+uprv_strlen(filename);
216    if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
217        *basename++=U_FILE_SEP_CHAR;
218    }
219
220    /* initialize */
221    init();
222
223    /* process the file */
224    uprv_strcpy(basename,inputFileName);
225    parseMappings(filename,FALSE, &errorCode);
226    if(U_FAILURE(errorCode)) {
227        fprintf(stderr, "Could not open file %s for reading. Error: %s \n", filename, u_errorName(errorCode));
228        return errorCode;
229    }
230
231    if(options[NORMALIZE].doesOccur){ /* this option might be set by @normalize;; in the source file */
232        /* set up directory for NormalizationCorrections.txt */
233        uprv_strcpy(filename,icuUniDataDir);
234        basename=filename+uprv_strlen(filename);
235        if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
236            *basename++=U_FILE_SEP_CHAR;
237        }
238
239        *basename++=U_FILE_SEP_CHAR;
240        uprv_strcpy(basename,NORM_CORRECTIONS_FILE_NAME);
241
242        parseNormalizationCorrections(filename,&errorCode);
243        if(U_FAILURE(errorCode)){
244            fprintf(stderr,"Could not open file %s for reading \n", filename);
245            return errorCode;
246        }
247        sprepOptions |= _SPREP_NORMALIZATION_ON;
248    }
249
250    if(options[CHECK_BIDI].doesOccur){ /* this option might be set by @check-bidi;; in the source file */
251        sprepOptions |= _SPREP_CHECK_BIDI_ON;
252    }
253
254    setOptions(sprepOptions);
255
256    /* process parsed data */
257    if(U_SUCCESS(errorCode)) {
258        /* write the data file */
259        generateData(destDir, bundleName);
260
261        cleanUpData();
262    }
263
264    uprv_free(filename);
265
266    u_cleanup();
267
268#endif
269
270    return errorCode;
271}
272
273#if !UCONFIG_NO_IDNA
274
275static void U_CALLCONV
276normalizationCorrectionsLineFn(void *context,
277                    char *fields[][2], int32_t fieldCount,
278                    UErrorCode *pErrorCode) {
279    uint32_t mapping[40];
280    char *end, *s;
281    uint32_t code;
282    int32_t length;
283    UVersionInfo version;
284    UVersionInfo thisVersion;
285
286    /* get the character code, field 0 */
287    code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
288    if(U_FAILURE(*pErrorCode)) {
289        fprintf(stderr, "gensprep: error parsing NormalizationCorrections.txt mapping at %s\n", fields[0][0]);
290        exit(*pErrorCode);
291    }
292    /* Original (erroneous) decomposition */
293    s = fields[1][0];
294
295    /* parse the mapping string */
296    length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);
297
298    /* ignore corrected decomposition */
299
300    u_versionFromString(version,fields[3][0] );
301    u_versionFromString(thisVersion, "3.2.0");
302
303
304
305    if(U_FAILURE(*pErrorCode)) {
306        fprintf(stderr, "gensprep error parsing NormalizationCorrections.txt of U+%04lx - %s\n",
307                (long)code, u_errorName(*pErrorCode));
308        exit(*pErrorCode);
309    }
310
311    /* store the mapping */
312    if( version[0] > thisVersion[0] ||
313        ((version[0]==thisVersion[0]) && (version[1] > thisVersion[1]))
314        ){
315        storeMapping(code,mapping, length, USPREP_MAP, pErrorCode);
316    }
317    setUnicodeVersionNC(version);
318}
319
320static void
321parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) {
322    char *fields[4][2];
323
324    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
325        return;
326    }
327
328    u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode);
329
330    /* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */
331
332    if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) {
333        fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
334        exit(*pErrorCode);
335    }
336}
337
338static void U_CALLCONV
339strprepProfileLineFn(void *context,
340              char *fields[][2], int32_t fieldCount,
341              UErrorCode *pErrorCode) {
342    uint32_t mapping[40];
343    char *end, *map;
344    uint32_t code;
345    int32_t length;
346   /*UBool* mapWithNorm = (UBool*) context;*/
347    const char* typeName;
348    uint32_t rangeStart=0,rangeEnd =0;
349    const char* filename = (const char*) context;
350    const char *s;
351
352    s = u_skipWhitespace(fields[0][0]);
353    if (*s == '@') {
354        /* special directive */
355        s++;
356        length = fields[0][1] - s;
357        if (length >= NORMALIZE_DIRECTIVE_LEN
358            && uprv_strncmp(s, NORMALIZE_DIRECTIVE, NORMALIZE_DIRECTIVE_LEN) == 0) {
359            options[NORMALIZE].doesOccur = TRUE;
360            return;
361        }
362        else if (length >= CHECK_BIDI_DIRECTIVE_LEN
363            && uprv_strncmp(s, CHECK_BIDI_DIRECTIVE, CHECK_BIDI_DIRECTIVE_LEN) == 0) {
364            options[CHECK_BIDI].doesOccur = TRUE;
365            return;
366        }
367        else {
368            fprintf(stderr, "gensprep error parsing a directive %s.", fields[0][0]);
369        }
370    }
371
372    typeName = fields[2][0];
373    map = fields[1][0];
374
375    if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){
376
377        u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
378        if(U_FAILURE(*pErrorCode)){
379            fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
380            return;
381        }
382
383        /* store the range */
384        storeRange(rangeStart,rangeEnd,USPREP_UNASSIGNED, pErrorCode);
385
386    }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){
387
388        u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
389        if(U_FAILURE(*pErrorCode)){
390            fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
391            return;
392        }
393
394        /* store the range */
395        storeRange(rangeStart,rangeEnd,USPREP_PROHIBITED, pErrorCode);
396
397    }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){
398
399        /* get the character code, field 0 */
400        code=(uint32_t)uprv_strtoul(s, &end, 16);
401        if(end<=s || end!=fields[0][1]) {
402            fprintf(stderr, "gensprep: syntax error in field 0 at %s\n", fields[0][0]);
403            *pErrorCode=U_PARSE_ERROR;
404            exit(U_PARSE_ERROR);
405        }
406
407        /* parse the mapping string */
408        length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode);
409
410        /* store the mapping */
411        storeMapping(code,mapping, length,USPREP_MAP, pErrorCode);
412
413    }else{
414        *pErrorCode = U_INVALID_FORMAT_ERROR;
415    }
416
417    if(U_FAILURE(*pErrorCode)) {
418        fprintf(stderr, "gensprep error parsing  %s line %s at %s. Error: %s\n",filename,
419               fields[0][0],fields[2][0],u_errorName(*pErrorCode));
420        exit(*pErrorCode);
421    }
422
423}
424
425static void
426parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode) {
427    char *fields[3][2];
428
429    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
430        return;
431    }
432
433    u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode);
434
435    /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/
436
437    if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
438        fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
439        exit(*pErrorCode);
440    }
441}
442
443
444#endif /* #if !UCONFIG_NO_IDNA */
445
446/*
447 * Hey, Emacs, please set the following:
448 *
449 * Local Variables:
450 * indent-tabs-mode: nil
451 * End:
452 *
453 */
454