1# Copyright (c) 2002-2010 International Business Machines Corporation and 2# others. All Rights Reserved. 3# 4# file: line_fi.txt 5# 6# Line Breaking Rules 7# Implement default line breaking as defined by 8# Unicode Standard Annex #14 Revision 24 for Unicode 6.0 9# http://www.unicode.org/reports/tr14/ 10# 11# TODO: Rule LB 8 remains as it was in Unicode 5.2 12# This is only because of a limitation of ICU break engine implementation, 13# not because the older behavior is desirable. 14 15# 16# Character Classes defined by TR 14. 17# 18 19!!chain; 20!!LBCMNoChain; 21 22 23!!lookAheadHardBreak; 24# 25# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere 26# and only used for the line break rules. 27# 28# It is used in the implementation of rule LB 10 29# which says to treat any combining mark that is not attached to a base 30# character as if it were of class AL (alphabetic). 31# 32# The problem occurs in the reverse rules. 33# 34# Consider a sequence like, with correct breaks as shown 35# LF ID CM AL AL 36# ^ ^ ^ 37# Then consider the sequence without the initial ID (ideographic) 38# LF CM AL AL 39# ^ ^ 40# Our CM, which in the first example was attached to the ideograph, 41# is now unattached, becomes an alpha, and joins in with the other 42# alphas. 43# 44# When iterating forwards, these sequences do not present any problems 45# When iterating backwards, we need to look ahead when encountering 46# a CM to see whether it attaches to something further on or not. 47# (Look-ahead in a reverse rule is looking towards the start) 48# 49# If the CM is unattached, we need to force a break. 50# 51# !!lookAheadHardBreak forces the run time state machine to 52# stop immediately when a look ahead rule ( '/' operator) matches, 53# and set the match position to that of the look-ahead operator, 54# no matter what other rules may be in play at the time. 55# 56# See rule LB 19 for an example. 57# 58 59$AI = [:LineBreak = Ambiguous:]; 60$AL = [:LineBreak = Alphabetic:]; 61$BA = [[:LineBreak = Break_After:] - [\u2010]]; 62$HH = [\u2010]; 63$BB = [:LineBreak = Break_Before:]; 64$BK = [:LineBreak = Mandatory_Break:]; 65$B2 = [:LineBreak = Break_Both:]; 66$CB = [:LineBreak = Contingent_Break:]; 67$CL = [:LineBreak = Close_Punctuation:]; 68$CM = [:LineBreak = Combining_Mark:]; 69$CP = [:LineBreak = Close_Parenthesis:]; 70$CR = [:LineBreak = Carriage_Return:]; 71$EX = [:LineBreak = Exclamation:]; 72$GL = [:LineBreak = Glue:]; 73$HY = [:LineBreak = Hyphen:]; 74$H2 = [:LineBreak = H2:]; 75$H3 = [:LineBreak = H3:]; 76$ID = [:LineBreak = Ideographic:]; 77$IN = [:LineBreak = Inseperable:]; 78$IS = [:LineBreak = Infix_Numeric:]; 79$JL = [:LineBreak = JL:]; 80$JV = [:LineBreak = JV:]; 81$JT = [:LineBreak = JT:]; 82$LF = [:LineBreak = Line_Feed:]; 83$NL = [:LineBreak = Next_Line:]; 84$NS = [:LineBreak = Nonstarter:]; 85$NU = [:LineBreak = Numeric:]; 86$OP = [:LineBreak = Open_Punctuation:]; 87$PO = [:LineBreak = Postfix_Numeric:]; 88$PR = [:LineBreak = Prefix_Numeric:]; 89$QU = [:LineBreak = Quotation:]; 90$SA = [:LineBreak = Complex_Context:]; 91$SG = [:LineBreak = Surrogate:]; 92$SP = [:LineBreak = Space:]; 93$SY = [:LineBreak = Break_Symbols:]; 94$WJ = [:LineBreak = Word_Joiner:]; 95$XX = [:LineBreak = Unknown:]; 96$ZW = [:LineBreak = ZWSpace:]; 97 98# Dictionary character set, for triggering language-based break engines. Currently 99# limited to LineBreak=Complex_Context. Note that this set only works in Unicode 100# 5.0 or later as the definition of Complex_Context was corrected to include all 101# characters requiring dictionary break. 102 103$dictionary = [:LineBreak = Complex_Context:]; 104 105# 106# Rule LB1. By default, treat AI (characters with ambiguous east Asian width), 107# SA (South East Asian: Thai, Lao, Khmer) 108# SG (Unpaired Surrogates) 109# XX (Unknown, unassigned) 110# as $AL (Alphabetic) 111# 112$ALPlus = [$AL $AI $SA $SG $XX]; 113 114# 115# Combining Marks. X $CM* behaves as if it were X. Rule LB6. 116# 117$ALcm = $ALPlus $CM*; 118$BAcm = $BA $CM*; 119$HHcm = $HH $CM*; 120$BBcm = $BB $CM*; 121$B2cm = $B2 $CM*; 122$CLcm = $CL $CM*; 123$CPcm = $CP $CM*; 124$EXcm = $EX $CM*; 125$GLcm = $GL $CM*; 126$HYcm = $HY $CM*; 127$H2cm = $H2 $CM*; 128$H3cm = $H3 $CM*; 129$IDcm = $ID $CM*; 130$INcm = $IN $CM*; 131$IScm = $IS $CM*; 132$JLcm = $JL $CM*; 133$JVcm = $JV $CM*; 134$JTcm = $JT $CM*; 135$NScm = $NS $CM*; 136$NUcm = $NU $CM*; 137$OPcm = $OP $CM*; 138$POcm = $PO $CM*; 139$PRcm = $PR $CM*; 140$QUcm = $QU $CM*; 141$SYcm = $SY $CM*; 142$WJcm = $WJ $CM*; 143 144## ------------------------------------------------- 145 146!!forward; 147 148# 149# Each class of character can stand by itself as an unbroken token, with trailing combining stuff 150# 151$ALPlus $CM+; 152$BA $CM+; 153$HH $CM+; 154$BB $CM+; 155$B2 $CM+; 156$CL $CM+; 157$CP $CM+; 158$EX $CM+; 159$GL $CM+; 160$HY $CM+; 161$H2 $CM+; 162$H3 $CM+; 163$ID $CM+; 164$IN $CM+; 165$IS $CM+; 166$JL $CM+; 167$JV $CM+; 168$JT $CM+; 169$NS $CM+; 170$NU $CM+; 171$OP $CM+; 172$PO $CM+; 173$PR $CM+; 174$QU $CM+; 175$SY $CM+; 176$WJ $CM+; 177 178# 179# CAN_CM is the set of characters that may combine with CM combining chars. 180# Note that Linebreak UAX 14's concept of a combining char and the rules 181# for what they can combine with are _very_ different from the rest of Unicode. 182# 183# Note that $CM itself is left out of this set. If CM is needed as a base 184# it must be listed separately in the rule. 185# 186$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs 187$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs 188 189# 190# AL_FOLLOW set of chars that can unconditionally follow an AL 191# Needed in rules where stand-alone $CM s are treated as AL. 192# Chaining is disabled with CM because it causes other failures, 193# so for this one case we need to manually list out longer sequences. 194# 195$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP]; 196$AL_FOLLOW_CM = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HH $HY $NS $IN $NU $ALPlus]; 197$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM]; 198 199 200# 201# Rule LB 4, 5 Mandatory (Hard) breaks. 202# 203$LB4Breaks = [$BK $CR $LF $NL]; 204$LB4NonBreaks = [^$BK $CR $LF $NL]; 205$CR $LF {100}; 206 207# 208# LB 6 Do not break before hard line breaks. 209# 210$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks. 211$CAN_CM $CM* $LB4Breaks {100}; 212$CM+ $LB4Breaks {100}; 213 214# LB 7 x SP 215# x ZW 216$LB4NonBreaks [$SP $ZW]; 217$CAN_CM $CM* [$SP $ZW]; 218$CM+ [$SP $ZW]; 219 220# 221# LB 8 Break after zero width space 222# TODO: ZW SP* <break> 223# An engine change is required to write the reverse rule for this. 224# For now, leave the Unicode 5.2 rule, ZW <break> 225# 226$LB8Breaks = [$LB4Breaks $ZW]; 227$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]]; 228 229 230# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL 231# $CM not covered by the above needs to behave like $AL 232# See definition of $CAN_CM. 233 234$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules. 235$CM+; 236 237# 238# LB 11 Do not break before or after WORD JOINER & related characters. 239# 240$CAN_CM $CM* $WJcm; 241$LB8NonBreaks $WJcm; 242$CM+ $WJcm; 243 244$WJcm $CANT_CM; 245$WJcm $CAN_CM $CM*; 246 247# 248# LB 12 Do not break after NBSP and related characters. 249# GL x 250# 251$GLcm $CAN_CM $CM*; 252$GLcm $CANT_CM; 253 254# 255# LB 12a Do not break before NBSP and related characters ... 256# [^SP BA HY] x GL 257# 258[[$LB8NonBreaks] - [$SP $BA $HH $HY]] $CM* $GLcm; 259$CM+ GLcm; 260 261 262 263# 264# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces. 265# 266$LB8NonBreaks $CL; 267$CAN_CM $CM* $CL; 268$CM+ $CL; # by rule 10, stand-alone CM behaves as AL 269 270$LB8NonBreaks $CP; 271$CAN_CM $CM* $CP; 272$CM+ $CP; # by rule 10, stand-alone CM behaves as AL 273 274$LB8NonBreaks $EX; 275$CAN_CM $CM* $EX; 276$CM+ $EX; # by rule 10, stand-alone CM behaves as AL 277 278$LB8NonBreaks $IS; 279$CAN_CM $CM* $IS; 280$CM+ $IS; # by rule 10, stand-alone CM behaves as AL 281 282$LB8NonBreaks $SY; 283$CAN_CM $CM* $SY; 284$CM+ $SY; # by rule 10, stand-alone CM behaves as AL 285 286 287# 288# LB 14 Do not break after OP, even after spaces 289# 290$OPcm $SP* $CAN_CM $CM*; 291$OPcm $SP* $CANT_CM; 292 293$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL 294 295# LB 15 296$QUcm $SP* $OPcm; 297 298# LB 16 299($CLcm | $CPcm) $SP* $NScm; 300 301# LB 17 302$B2cm $SP* $B2cm; 303 304# 305# LB 18 Break after spaces. 306# 307$LB18NonBreaks = [$LB8NonBreaks - [$SP]]; 308$LB18Breaks = [$LB8Breaks $SP]; 309 310 311# LB 19 312# x QU 313$LB18NonBreaks $CM* $QUcm; 314$CM+ $QUcm; 315 316# QU x 317$QUcm .?; 318$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc. 319 # TODO: I don't think this rule is needed. 320 321 322# LB 20 323# <break> $CB 324# $CB <break> 325 326$LB20NonBreaks = [$LB18NonBreaks - $CB]; 327 328# LB 20.09 added rule for Finnish tailoring 329# LB 21 x (BA | HY | NS) 330# BB x 331# 332$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm) / $AL; 333$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm); 334($HY | $HH) $AL; 335 336$BBcm [^$CB]; # $BB x 337$BBcm $LB20NonBreaks $CM*; 338 339# LB 22 340$ALcm $INcm; 341$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL 342$IDcm $INcm; 343$INcm $INcm; 344$NUcm $INcm; 345 346 347# $LB 23 348$IDcm $POcm; 349$ALcm $NUcm; # includes $LB19 350$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL 351$NUcm $ALcm; 352 353# 354# LB 24 355# 356$PRcm $IDcm; 357$PRcm $ALcm; 358$POcm $ALcm; 359 360# 361# LB 25 Numbers. 362# 363($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?; 364 365# LB 26 Do not break a Korean syllable 366# 367$JLcm ($JLcm | $JVcm | $H2cm | $H3cm); 368($JVcm | $H2cm) ($JVcm | $JTcm); 369($JTcm | $H3cm) $JTcm; 370 371# LB 27 Treat korean Syllable Block the same as ID (don't break it) 372($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm; 373($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm; 374$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm); 375 376 377# LB 28 Do not break between alphabetics 378# 379$ALcm $ALcm; 380$CM+ $ALcm; # The $CM+ is from rule 10, an unattached CM is treated as AL 381 382# LB 29 383$IScm $ALcm; 384 385# LB 30 386($ALcm | $NUcm) $OPcm; 387$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL. 388$CPcm ($ALcm | $NUcm); 389 390 391# 392# Reverse Rules. 393# 394## ------------------------------------------------- 395 396!!reverse; 397 398$CM+ $ALPlus; 399$CM+ $BA; 400$CM+ $HH; 401$CM+ $BB; 402$CM+ $B2; 403$CM+ $CL; 404$CM+ $CP; 405$CM+ $EX; 406$CM+ $GL; 407$CM+ $HY; 408$CM+ $H2; 409$CM+ $H3; 410$CM+ $ID; 411$CM+ $IN; 412$CM+ $IS; 413$CM+ $JL; 414$CM+ $JV; 415$CM+ $JT; 416$CM+ $NS; 417$CM+ $NU; 418$CM+ $OP; 419$CM+ $PO; 420$CM+ $PR; 421$CM+ $QU; 422$CM+ $SY; 423$CM+ $WJ; 424$CM+; 425 426 427# 428# Sequences of the form (shown forwards) 429# [CANT_CM] <break> [CM] [whatever] 430# The CM needs to behave as an AL 431# 432$AL_FOLLOW $CM+ / ( 433 [$BK $CR $LF $NL $ZW {eof}] | 434 $SP+ $CM+ $SP | 435 $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break. 436 # LB14 says OP SP* x . 437 # becomes OP SP* x AL 438 # becomes OP SP* x CM+ AL_FOLLOW 439 # 440 # Further note: the $AL in [$AL {eof}] is only to work around 441 # a rule compiler bug which complains about 442 # empty sets otherwise. 443 444# 445# Sequences of the form (shown forwards) 446# [CANT_CM] <break> [CM] <break> [PR] 447# The CM needs to behave as an AL 448# This rule is concerned about getting the second of the two <breaks> in place. 449# 450 451[$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}]; 452 453 454 455# LB 4, 5, 5 456 457$LB4Breaks [$LB4NonBreaks-$CM]; 458$LB4Breaks $CM+ $CAN_CM; 459$LF $CR; 460 461 462# LB 7 x SP 463# x ZW 464[$SP $ZW] [$LB4NonBreaks-$CM]; 465[$SP $ZW] $CM+ $CAN_CM; 466 467# LB 8 ZW SP* <break> 468# TODO: to implement this, we need more than one look-ahead hard break in play at a time. 469# Requires an engine enhancement. 470# / $SP* $ZW 471 472# LB 9,10 Combining marks. 473# X $CM needs to behave like X, where X is not $SP or controls. 474# $CM not covered by the above needs to behave like $AL 475# Stick together any combining sequences that don't match other rules. 476$CM+ $CAN_CM; 477 478 479# LB 11 480$CM* $WJ $CM* $CAN_CM; 481$CM* $WJ [$LB8NonBreaks-$CM]; 482 483 $CANT_CM $CM* $WJ; 484$CM* $CAN_CM $CM* $WJ; 485 486# LB 12a 487# [^SP BA HY] x GL 488# 489$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]]; 490 491# LB 12 492# GL x 493# 494$CANT_CM $CM* $GL; 495$CM* $CAN_CM $CM* $GL; 496 497 498# LB 13 499$CL $CM+ $CAN_CM; 500$CP $CM+ $CAN_CM; 501$EX $CM+ $CAN_CM; 502$IS $CM+ $CAN_CM; 503$SY $CM+ $CAN_CM; 504 505$CL [$LB8NonBreaks-$CM]; 506$CP [$LB8NonBreaks-$CM]; 507$EX [$LB8NonBreaks-$CM]; 508$IS [$LB8NonBreaks-$CM]; 509$SY [$LB8NonBreaks-$CM]; 510 511# Rule 13 & 14 taken together for an edge case. 512# Match this, shown forward 513# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY) 514# This really wants to chain at the $CM+ (which is acting as an $AL) 515# except for $CM chaining being disabled. 516[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP; 517 518# LB 14 OP SP* x 519# 520$CM* $CAN_CM $SP* $CM* $OP; 521 $CANT_CM $SP* $CM* $OP; 522$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP 523 524 $AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP; 525$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP; 526$SY $CM $SP+ $OP; # TODO: Experiment. Remove. 527 528 529 530# LB 15 531$CM* $OP $SP* $CM* $QU; 532 533# LB 16 534$CM* $NS $SP* $CM* ($CL | $CP); 535 536# LB 17 537$CM* $B2 $SP* $CM* $B2; 538 539# LB 18 break after spaces 540# Nothing explicit needed here. 541 542 543# 544# LB 19 545# 546$CM* $QU $CM* $CAN_CM; # . x QU 547$CM* $QU $LB18NonBreaks; 548 549 550$CM* $CAN_CM $CM* $QU; # QU x . 551 $CANT_CM $CM* $QU; 552 553# 554# LB 20 Break before and after CB. 555# nothing needed here. 556# 557 558# LB 20.09 added rule for Finnish tailoring 559$AL ($HY | $HH) / $SP; 560 561# LB 21 562$CM* ($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS) 563 564$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x . 565[^$CB] $CM* $BB; # 566 567 568 569# LB 22 570$CM* $IN $CM* $ALPlus; 571$CM* $IN $CM* $ID; 572$CM* $IN $CM* $IN; 573$CM* $IN $CM* $NU; 574 575# LB 23 576$CM* $PO $CM* $ID; 577$CM* $NU $CM* $ALPlus; 578$CM* $ALPlus $CM* $NU; 579 580# LB 24 581$CM* $ID $CM* $PR; 582$CM* $ALPlus $CM* $PR; 583$CM* $ALPlus $CM* $PO; 584 585 586# LB 25 587($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?; 588 589# LB 26 590$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL; 591$CM* ($JT | $JV) $CM* ($H2 | $JV); 592$CM* $JT $CM* ($H3 | $JT); 593 594# LB 27 595$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL); 596$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL); 597$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR; 598 599# LB 28 600$CM* $ALPlus $CM* $ALPlus; 601 602 603# LB 29 604$CM* $ALPlus $CM* $IS; 605 606# LB 30 607$CM* $OP $CM* ($ALPlus | $NU); 608$CM* ($ALPlus | $NU) $CM* $CP; 609 610 611## ------------------------------------------------- 612 613!!safe_reverse; 614 615# LB 9 616$CM+ [^$CM $BK $CR $LF $NL $ZW $SP]; 617$CM+ $SP / .; 618 619# LB 14 620$SP+ $CM* $OP; 621 622# LB 15 623$SP+ $CM* $QU; 624 625# LB 16 626$SP+ $CM* ($CL | $CP); 627 628# LB 17 629$SP+ $CM* $B2; 630 631# LB 25 632($CM* ($IS | $SY))+ $CM* $NU; 633($CL | $CP) $CM* ($NU | $IS | $SY); 634 635# For dictionary-based break 636$dictionary $dictionary; 637 638## ------------------------------------------------- 639 640!!safe_forward; 641 642# Skip forward over all character classes that are involved in 643# rules containing patterns with possibly more than one char 644# of context. 645# 646# It might be slightly more efficient to have specific rules 647# instead of one generic one, but only if we could 648# turn off rule chaining. We don't want to move more 649# than necessary. 650# 651[$CM $OP $QU $CL $CP $B2 $PR $HY $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $dictionary]; 652$dictionary $dictionary; 653 654