1// Copyright (C) 2016 and later: Unicode, Inc. and others. 2// License & terms of use: http://www.unicode.org/copyright.html 3/* 4******************************************************************************* 5* 6* Copyright (C) 2003-2016, International Business Machines 7* Corporation and others. All Rights Reserved. 8* 9******************************************************************************* 10* file name: gensprep.c 11* encoding: US-ASCII 12* tab size: 8 (not used) 13* indentation:4 14* 15* created on: 2003-02-06 16* created by: Ram Viswanadha 17* 18* This program reads the Profile.txt files, 19* parses them, and extracts the data for StringPrep profile. 20* It then preprocesses it and writes a binary file for efficient use 21* in various StringPrep conversion processes. 22*/ 23 24#define USPREP_TYPE_NAMES_ARRAY 1 25 26#include <stdio.h> 27#include <stdlib.h> 28 29#include "cmemory.h" 30#include "cstring.h" 31#include "unewdata.h" 32#include "uoptions.h" 33#include "uparse.h" 34#include "sprpimpl.h" 35 36#include "unicode/uclean.h" 37#include "unicode/udata.h" 38#include "unicode/utypes.h" 39#include "unicode/putil.h" 40 41 42U_CDECL_BEGIN 43#include "gensprep.h" 44U_CDECL_END 45 46UBool beVerbose=FALSE, haveCopyright=TRUE; 47 48#define NORM_CORRECTIONS_FILE_NAME "NormalizationCorrections.txt" 49 50#define NORMALIZE_DIRECTIVE "normalize" 51#define NORMALIZE_DIRECTIVE_LEN 9 52#define CHECK_BIDI_DIRECTIVE "check-bidi" 53#define CHECK_BIDI_DIRECTIVE_LEN 10 54 55/* prototypes --------------------------------------------------------------- */ 56 57static void 58parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode); 59 60static void 61parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode); 62 63 64/* -------------------------------------------------------------------------- */ 65 66static UOption options[]={ 67 UOPTION_HELP_H, 68 UOPTION_HELP_QUESTION_MARK, 69 UOPTION_VERBOSE, 70 UOPTION_COPYRIGHT, 71 UOPTION_DESTDIR, 72 UOPTION_SOURCEDIR, 73 UOPTION_ICUDATADIR, 74 UOPTION_BUNDLE_NAME, 75 { "normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0 }, 76 { "norm-correction", NULL, NULL, NULL, 'm', UOPT_REQUIRES_ARG, 0 }, 77 { "check-bidi", NULL, NULL, NULL, 'k', UOPT_NO_ARG, 0}, 78 { "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 }, 79}; 80 81enum{ 82 HELP, 83 HELP_QUESTION_MARK, 84 VERBOSE, 85 COPYRIGHT, 86 DESTDIR, 87 SOURCEDIR, 88 ICUDATADIR, 89 BUNDLE_NAME, 90 NORMALIZE, 91 NORM_CORRECTION_DIR, 92 CHECK_BIDI, 93 UNICODE_VERSION 94}; 95 96static int printHelp(int argc, char* argv[]){ 97 /* 98 * Broken into chucks because the C89 standard says the minimum 99 * required supported string length is 509 bytes. 100 */ 101 fprintf(stderr, 102 "Usage: %s [-options] [file_name]\n" 103 "\n" 104 "Read the files specified and\n" 105 "create a binary file [package-name]_[bundle-name]." DATA_TYPE " with the StringPrep profile data\n" 106 "\n", 107 argv[0]); 108 fprintf(stderr, 109 "Options:\n" 110 "\t-h or -? or --help print this usage text\n" 111 "\t-v or --verbose verbose output\n" 112 "\t-c or --copyright include a copyright notice\n"); 113 fprintf(stderr, 114 "\t-d or --destdir destination directory, followed by the path\n" 115 "\t-s or --sourcedir source directory of ICU data, followed by the path\n" 116 "\t-b or --bundle-name generate the ouput data file with the name specified\n" 117 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" 118 "\t followed by path, defaults to %s\n", 119 u_getDataDirectory()); 120 fprintf(stderr, 121 "\t-n or --normalize turn on the option for normalization and include mappings\n" 122 "\t from NormalizationCorrections.txt from the given path,\n" 123 "\t e.g: /test/icu/source/data/unidata\n"); 124 fprintf(stderr, 125 "\t-m or --norm-correction use NormalizationCorrections.txt from the given path\n" 126 "\t when the input file contains a normalization directive.\n" 127 "\t unlike -n/--normalize, this option does not force the\n" 128 "\t normalization.\n"); 129 fprintf(stderr, 130 "\t-k or --check-bidi turn on the option for checking for BiDi in the profile\n" 131 "\t-u or --unicode version of Unicode to be used with this profile followed by the version\n" 132 ); 133 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; 134} 135 136 137extern int 138main(int argc, char* argv[]) { 139#if !UCONFIG_NO_IDNA 140 char* filename = NULL; 141#endif 142 const char *srcDir=NULL, *destDir=NULL, *icuUniDataDir=NULL; 143 const char *bundleName=NULL, *inputFileName = NULL; 144 char *basename=NULL; 145 int32_t sprepOptions = 0; 146 147 UErrorCode errorCode=U_ZERO_ERROR; 148 149 U_MAIN_INIT_ARGS(argc, argv); 150 151 /* preset then read command line options */ 152 options[DESTDIR].value=u_getDataDirectory(); 153 options[SOURCEDIR].value=""; 154 options[UNICODE_VERSION].value="0"; /* don't assume the unicode version */ 155 options[BUNDLE_NAME].value = DATA_NAME; 156 options[NORMALIZE].value = ""; 157 158 argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options); 159 160 /* error handling, printing usage message */ 161 if(argc<0) { 162 fprintf(stderr, 163 "error in command line argument \"%s\"\n", 164 argv[-argc]); 165 } 166 if(argc<0 || options[HELP].doesOccur || options[HELP_QUESTION_MARK].doesOccur) { 167 return printHelp(argc, argv); 168 169 } 170 171 /* get the options values */ 172 beVerbose=options[VERBOSE].doesOccur; 173 haveCopyright=options[COPYRIGHT].doesOccur; 174 srcDir=options[SOURCEDIR].value; 175 destDir=options[DESTDIR].value; 176 bundleName = options[BUNDLE_NAME].value; 177 if(options[NORMALIZE].doesOccur) { 178 icuUniDataDir = options[NORMALIZE].value; 179 } else { 180 icuUniDataDir = options[NORM_CORRECTION_DIR].value; 181 } 182 183 if(argc<2) { 184 /* print the help message */ 185 return printHelp(argc, argv); 186 } else { 187 inputFileName = argv[1]; 188 } 189 if(!options[UNICODE_VERSION].doesOccur){ 190 return printHelp(argc, argv); 191 } 192 if(options[ICUDATADIR].doesOccur) { 193 u_setDataDirectory(options[ICUDATADIR].value); 194 } 195#if UCONFIG_NO_IDNA 196 197 fprintf(stderr, 198 "gensprep writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE 199 " because UCONFIG_NO_IDNA is set, \n" 200 "see icu/source/common/unicode/uconfig.h\n"); 201 generateData(destDir, bundleName); 202 203#else 204 205 setUnicodeVersion(options[UNICODE_VERSION].value); 206 filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + uprv_strlen(inputFileName) + (icuUniDataDir == NULL ? 0 : uprv_strlen(icuUniDataDir)) + 40); /* hopefully this should be enough */ 207 208 /* prepare the filename beginning with the source dir */ 209 if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL && uprv_strchr(srcDir,U_FILE_ALT_SEP_CHAR) == NULL){ 210 filename[0] = '.'; 211 filename[1] = U_FILE_SEP_CHAR; 212 uprv_strcpy(filename+2,srcDir); 213 }else{ 214 uprv_strcpy(filename, srcDir); 215 } 216 217 basename=filename+uprv_strlen(filename); 218 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { 219 *basename++=U_FILE_SEP_CHAR; 220 } 221 222 /* initialize */ 223 init(); 224 225 /* process the file */ 226 uprv_strcpy(basename,inputFileName); 227 parseMappings(filename,FALSE, &errorCode); 228 if(U_FAILURE(errorCode)) { 229 fprintf(stderr, "Could not open file %s for reading. Error: %s \n", filename, u_errorName(errorCode)); 230 return errorCode; 231 } 232 233 if(options[NORMALIZE].doesOccur){ /* this option might be set by @normalize;; in the source file */ 234 /* set up directory for NormalizationCorrections.txt */ 235 uprv_strcpy(filename,icuUniDataDir); 236 basename=filename+uprv_strlen(filename); 237 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { 238 *basename++=U_FILE_SEP_CHAR; 239 } 240 241 *basename++=U_FILE_SEP_CHAR; 242 uprv_strcpy(basename,NORM_CORRECTIONS_FILE_NAME); 243 244 parseNormalizationCorrections(filename,&errorCode); 245 if(U_FAILURE(errorCode)){ 246 fprintf(stderr,"Could not open file %s for reading \n", filename); 247 return errorCode; 248 } 249 sprepOptions |= _SPREP_NORMALIZATION_ON; 250 } 251 252 if(options[CHECK_BIDI].doesOccur){ /* this option might be set by @check-bidi;; in the source file */ 253 sprepOptions |= _SPREP_CHECK_BIDI_ON; 254 } 255 256 setOptions(sprepOptions); 257 258 /* process parsed data */ 259 if(U_SUCCESS(errorCode)) { 260 /* write the data file */ 261 generateData(destDir, bundleName); 262 263 cleanUpData(); 264 } 265 266 uprv_free(filename); 267 268 u_cleanup(); 269 270#endif 271 272 return errorCode; 273} 274 275#if !UCONFIG_NO_IDNA 276 277static void U_CALLCONV 278normalizationCorrectionsLineFn(void *context, 279 char *fields[][2], int32_t fieldCount, 280 UErrorCode *pErrorCode) { 281 uint32_t mapping[40]; 282 char *end, *s; 283 uint32_t code; 284 int32_t length; 285 UVersionInfo version; 286 UVersionInfo thisVersion; 287 288 /* get the character code, field 0 */ 289 code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16); 290 if(U_FAILURE(*pErrorCode)) { 291 fprintf(stderr, "gensprep: error parsing NormalizationCorrections.txt mapping at %s\n", fields[0][0]); 292 exit(*pErrorCode); 293 } 294 /* Original (erroneous) decomposition */ 295 s = fields[1][0]; 296 297 /* parse the mapping string */ 298 length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode); 299 300 /* ignore corrected decomposition */ 301 302 u_versionFromString(version,fields[3][0] ); 303 u_versionFromString(thisVersion, "3.2.0"); 304 305 306 307 if(U_FAILURE(*pErrorCode)) { 308 fprintf(stderr, "gensprep error parsing NormalizationCorrections.txt of U+%04lx - %s\n", 309 (long)code, u_errorName(*pErrorCode)); 310 exit(*pErrorCode); 311 } 312 313 /* store the mapping */ 314 if( version[0] > thisVersion[0] || 315 ((version[0]==thisVersion[0]) && (version[1] > thisVersion[1])) 316 ){ 317 storeMapping(code,mapping, length, USPREP_MAP, pErrorCode); 318 } 319 setUnicodeVersionNC(version); 320} 321 322static void 323parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) { 324 char *fields[4][2]; 325 326 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { 327 return; 328 } 329 330 u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode); 331 332 /* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */ 333 334 if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) { 335 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); 336 exit(*pErrorCode); 337 } 338} 339 340static void U_CALLCONV 341strprepProfileLineFn(void *context, 342 char *fields[][2], int32_t fieldCount, 343 UErrorCode *pErrorCode) { 344 uint32_t mapping[40]; 345 char *end, *map; 346 uint32_t code; 347 int32_t length; 348 /*UBool* mapWithNorm = (UBool*) context;*/ 349 const char* typeName; 350 uint32_t rangeStart=0,rangeEnd =0; 351 const char* filename = (const char*) context; 352 const char *s; 353 354 s = u_skipWhitespace(fields[0][0]); 355 if (*s == '@') { 356 /* special directive */ 357 s++; 358 length = fields[0][1] - s; 359 if (length >= NORMALIZE_DIRECTIVE_LEN 360 && uprv_strncmp(s, NORMALIZE_DIRECTIVE, NORMALIZE_DIRECTIVE_LEN) == 0) { 361 options[NORMALIZE].doesOccur = TRUE; 362 return; 363 } 364 else if (length >= CHECK_BIDI_DIRECTIVE_LEN 365 && uprv_strncmp(s, CHECK_BIDI_DIRECTIVE, CHECK_BIDI_DIRECTIVE_LEN) == 0) { 366 options[CHECK_BIDI].doesOccur = TRUE; 367 return; 368 } 369 else { 370 fprintf(stderr, "gensprep error parsing a directive %s.", fields[0][0]); 371 } 372 } 373 374 typeName = fields[2][0]; 375 map = fields[1][0]; 376 377 if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){ 378 379 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode); 380 if(U_FAILURE(*pErrorCode)){ 381 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode)); 382 return; 383 } 384 385 /* store the range */ 386 storeRange(rangeStart,rangeEnd,USPREP_UNASSIGNED, pErrorCode); 387 388 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){ 389 390 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode); 391 if(U_FAILURE(*pErrorCode)){ 392 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode)); 393 return; 394 } 395 396 /* store the range */ 397 storeRange(rangeStart,rangeEnd,USPREP_PROHIBITED, pErrorCode); 398 399 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){ 400 401 /* get the character code, field 0 */ 402 code=(uint32_t)uprv_strtoul(s, &end, 16); 403 if(end<=s || end!=fields[0][1]) { 404 fprintf(stderr, "gensprep: syntax error in field 0 at %s\n", fields[0][0]); 405 *pErrorCode=U_PARSE_ERROR; 406 exit(U_PARSE_ERROR); 407 } 408 409 /* parse the mapping string */ 410 length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode); 411 412 /* store the mapping */ 413 storeMapping(code,mapping, length,USPREP_MAP, pErrorCode); 414 415 }else{ 416 *pErrorCode = U_INVALID_FORMAT_ERROR; 417 } 418 419 if(U_FAILURE(*pErrorCode)) { 420 fprintf(stderr, "gensprep error parsing %s line %s at %s. Error: %s\n",filename, 421 fields[0][0],fields[2][0],u_errorName(*pErrorCode)); 422 exit(*pErrorCode); 423 } 424 425} 426 427static void 428parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode) { 429 char *fields[3][2]; 430 431 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { 432 return; 433 } 434 435 u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode); 436 437 /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/ 438 439 if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) { 440 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); 441 exit(*pErrorCode); 442 } 443} 444 445 446#endif /* #if !UCONFIG_NO_IDNA */ 447 448/* 449 * Hey, Emacs, please set the following: 450 * 451 * Local Variables: 452 * indent-tabs-mode: nil 453 * End: 454 * 455 */ 456