1# Copyright (C) 2016 and later: Unicode, Inc. and others. 2# License & terms of use: http://www.unicode.org/copyright.html 3# Copyright (c) 2002-2016 International Business Machines Corporation and 4# others. All Rights Reserved. 5# 6# file: line.txt 7# 8# Line Breaking Rules 9# Implement default line breaking as defined by 10# Unicode Standard Annex #14 Revision 44 for Unicode 13.0 11# http://www.unicode.org/reports/tr14/, with the following modification: 12# 13# Boundaries between hyphens and following letters are suppressed when 14# there is a boundary preceding the hyphen. See rule 20.9 15# 16# This corresponds to CSS line-break=strict (BCP47 -u-lb-strict). 17# It sets characters of class CJ to behave like NS. 18 19# 20# Character Classes defined by TR 14. 21# 22 23!!chain; 24!!quoted_literals_only; 25 26$AI = [:LineBreak = Ambiguous:]; 27$AL = [:LineBreak = Alphabetic:]; 28$BA = [:LineBreak = Break_After:]; 29$HH = [\u2010]; # \u2010 is HYPHEN, default line break is BA. 30$BB = [:LineBreak = Break_Before:]; 31$BK = [:LineBreak = Mandatory_Break:]; 32$B2 = [:LineBreak = Break_Both:]; 33$CB = [:LineBreak = Contingent_Break:]; 34$CJ = [:LineBreak = Conditional_Japanese_Starter:]; 35$CL = [:LineBreak = Close_Punctuation:]; 36# $CM = [:LineBreak = Combining_Mark:]; 37$CP = [:LineBreak = Close_Parenthesis:]; 38$CR = [:LineBreak = Carriage_Return:]; 39$EB = [:LineBreak = EB:]; 40$EM = [:LineBreak = EM:]; 41$EX = [:LineBreak = Exclamation:]; 42$GL = [:LineBreak = Glue:]; 43$HL = [:LineBreak = Hebrew_Letter:]; 44$HY = [:LineBreak = Hyphen:]; 45$H2 = [:LineBreak = H2:]; 46$H3 = [:LineBreak = H3:]; 47$ID = [:LineBreak = Ideographic:]; 48$IN = [:LineBreak = Inseperable:]; 49$IS = [:LineBreak = Infix_Numeric:]; 50$JL = [:LineBreak = JL:]; 51$JV = [:LineBreak = JV:]; 52$JT = [:LineBreak = JT:]; 53$LF = [:LineBreak = Line_Feed:]; 54$NL = [:LineBreak = Next_Line:]; 55# NS includes CJ for CSS strict line breaking. 56$NS = [[:LineBreak = Nonstarter:] $CJ]; 57$NU = [:LineBreak = Numeric:]; 58$OP = [:LineBreak = Open_Punctuation:]; 59$PO = [:LineBreak = Postfix_Numeric:]; 60$PR = [:LineBreak = Prefix_Numeric:]; 61$QU = [:LineBreak = Quotation:]; 62$RI = [:LineBreak = Regional_Indicator:]; 63$SA = [:LineBreak = Complex_Context:]; 64$SG = [:LineBreak = Surrogate:]; 65$SP = [:LineBreak = Space:]; 66$SY = [:LineBreak = Break_Symbols:]; 67$WJ = [:LineBreak = Word_Joiner:]; 68$XX = [:LineBreak = Unknown:]; 69$ZW = [:LineBreak = ZWSpace:]; 70$ZWJ = [:LineBreak = ZWJ:]; 71 72# OP30 and CP30 are variants of OP and CP that appear in-line in rule LB30 from UAX 14, 73# without a formal name. Because ICU rules require multiple uses of the expressions, 74# give them a single definition with a name 75 76$OP30 = [$OP - [\p{ea=F}\p{ea=W}\p{ea=H}]]; 77$CP30 = [$CP - [\p{ea=F}\p{ea=W}\p{ea=H}]]; 78 79# By LB9, a ZWJ also behaves as a CM. Including it in the definition of CM avoids having to explicitly 80# list it in the numerous rules that use CM. 81# By LB1, SA characters with general categor of Mn or Mc also resolve to CM. 82 83$CM = [[:LineBreak = Combining_Mark:] $ZWJ [$SA & [[:Mn:][:Mc:]]]]; 84$CMX = [[$CM] - [$ZWJ]]; 85 86# Dictionary character set, for triggering language-based break engines. Currently 87# limited to LineBreak=Complex_Context (SA). 88 89$dictionary = [$SA]; 90 91# 92# Rule LB1. By default, treat AI (characters with ambiguous east Asian width), 93# SA (Dictionary chars, excluding Mn and Mc) 94# SG (Unpaired Surrogates) 95# XX (Unknown, unassigned) 96# as $AL (Alphabetic) 97# 98$ALPlus = [$AL $AI $SG $XX [$SA-[[:Mn:][:Mc:]]]]; 99 100 101## ------------------------------------------------- 102 103# 104# CAN_CM is the set of characters that may combine with CM combining chars. 105# Note that Linebreak UAX 14's concept of a combining char and the rules 106# for what they can combine with are _very_ different from the rest of Unicode. 107# 108# Note that $CM itself is left out of this set. If CM is needed as a base 109# it must be listed separately in the rule. 110# 111$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs 112$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs 113 114# 115# AL_FOLLOW set of chars that can unconditionally follow an AL 116# Needed in rules where stand-alone $CM s are treated as AL. 117# 118$AL_FOLLOW = [$BK $CR $LF $NL $ZW $SP $CL $CP $EX $HL $IS $SY $WJ $GL $OP30 $QU $BA $HY $NS $IN $NU $PR $PO $ALPlus]; 119 120 121# 122# Rule LB 4, 5 Mandatory (Hard) breaks. 123# 124$LB4Breaks = [$BK $CR $LF $NL]; 125$LB4NonBreaks = [^$BK $CR $LF $NL $CM]; 126$CR $LF {100}; 127 128# 129# LB 6 Do not break before hard line breaks. 130# 131$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks. 132$CAN_CM $CM* $LB4Breaks {100}; 133^$CM+ $LB4Breaks {100}; 134 135# LB 7 x SP 136# x ZW 137$LB4NonBreaks [$SP $ZW]; 138$CAN_CM $CM* [$SP $ZW]; 139^$CM+ [$SP $ZW]; 140 141# 142# LB 8 Break after zero width space 143# ZW SP* ÷ 144# 145$LB8Breaks = [$LB4Breaks $ZW]; 146$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]]; 147$ZW $SP* / [^$SP $ZW $LB4Breaks]; 148 149# LB 8a ZWJ x Do not break Emoji ZWJ sequences. 150# 151$ZWJ [^$CM]; 152 153# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL 154# $CM not covered by the above needs to behave like $AL 155# See definition of $CAN_CM. 156 157$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules. 158^$CM+; 159 160# 161# LB 11 Do not break before or after WORD JOINER & related characters. 162# 163$CAN_CM $CM* $WJ; 164$LB8NonBreaks $WJ; 165^$CM+ $WJ; 166 167$WJ $CM* .; 168 169# 170# LB 12 Do not break after NBSP and related characters. 171# GL x 172# 173$GL $CM* .; 174 175# 176# LB 12a Do not break before NBSP and related characters ... 177# [^SP BA HY] x GL 178# 179[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GL; 180^$CM+ $GL; 181 182 183 184 185# LB 13 Don't break before ']' or '!' or or '/', even after spaces. 186# 187$LB8NonBreaks $CL; 188$CAN_CM $CM* $CL; 189^$CM+ $CL; # by rule 10, stand-alone CM behaves as AL 190 191$LB8NonBreaks $CP; 192$CAN_CM $CM* $CP; 193^$CM+ $CP; # by rule 10, stand-alone CM behaves as AL 194 195$LB8NonBreaks $EX; 196$CAN_CM $CM* $EX; 197^$CM+ $EX; # by rule 10, stand-alone CM behaves as AL 198 199$LB8NonBreaks $SY; 200$CAN_CM $CM* $SY; 201^$CM+ $SY; # by rule 10, stand-alone CM behaves as AL 202 203 204# 205# LB 14 Do not break after OP, even after spaces 206# Note subtle interaction with "SP IS /" rules in LB14a. 207# This rule consumes the SP, chaining happens on the IS, effectivley overriding the SP IS rules, 208# which is the desired behavior. 209# 210$OP $CM* $SP* .; 211 212$OP $CM* $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL 213 # by rule 8, CM following a SP is stand-alone. 214 215 216# LB 14a Force a break before start of a number with a leading decimal pt, e.g. " .23" 217# Note: would be simpler to express as "$SP / $IS $CM* $NU;", but ICU rules have limitations. 218# See issue ICU-20303 219 220 221$CanFollowIS = [$BK $CR $LF $NL $SP $ZW $WJ $GL $CL $CP $EX $IS $SY $QU $BA $HY $NS $ALPlus $HL $IN]; 222$SP $IS / [^ $CanFollowIS $NU $CM]; 223$SP $IS $CM* $CMX / [^ $CanFollowIS $NU $CM]; 224 225# 226# LB 14b Do not break before numeric separators (IS), even after spaces. 227 228[$LB8NonBreaks - $SP] $IS; 229$SP $IS $CM* [$CanFollowIS {eof}]; 230$SP $IS $CM* $ZWJ [^$CM $NU]; 231 232$CAN_CM $CM* $IS; 233^$CM+ $IS; # by rule 10, stand-alone CM behaves as AL 234 235 236# LB 15 237$QU $CM* $SP* $OP; 238 239# LB 16 240($CL | $CP) $CM* $SP* $NS; 241 242# LB 17 243$B2 $CM* $SP* $B2; 244 245# 246# LB 18 Break after spaces. 247# 248$LB18NonBreaks = [$LB8NonBreaks - [$SP]]; 249$LB18Breaks = [$LB8Breaks $SP]; 250 251 252# LB 19 253# x QU 254$LB18NonBreaks $CM* $QU; 255^$CM+ $QU; 256 257# QU x 258$QU $CM* .; 259 260# LB 20 261# <break> $CB 262# $CB <break> 263# 264$LB20NonBreaks = [$LB18NonBreaks - $CB]; 265 266# LB 20.09 Don't break between Hyphens and Letters when there is a break preceding the hyphen. 267# Originally added as a Finnish tailoring, now promoted to default ICU behavior. 268# Note: this is not default UAX-14 behaviour. See issue ICU-8151. 269# 270^($HY | $HH) $CM* $ALPlus; 271 272# LB 21 x (BA | HY | NS) 273# BB x 274# 275$LB20NonBreaks $CM* ($BA | $HY | $NS); 276 277 278^$CM+ ($BA | $HY | $NS); 279 280$BB $CM* [^$CB]; # $BB x 281$BB $CM* $LB20NonBreaks; 282 283# LB 21a Don't break after Hebrew + Hyphen 284# HL (HY | BA) x 285# 286$HL $CM* ($HY | $BA) $CM* [^$CB]?; 287 288# LB 21b (forward) Don't break between SY and HL 289# (break between HL and SY already disallowed by LB 13 above) 290$SY $CM* $HL; 291 292# LB 22 Do not break before ellipses 293# 294$LB20NonBreaks $CM* $IN; 295^$CM+ $IN; 296 297 298# LB 23 299# 300($ALPlus | $HL) $CM* $NU; 301^$CM+ $NU; # Rule 10, any otherwise unattached CM behaves as AL 302$NU $CM* ($ALPlus | $HL); 303 304# LB 23a 305# 306$PR $CM* ($ID | $EB | $EM); 307($ID | $EB | $EM) $CM* $PO; 308 309 310# 311# LB 24 312# 313($PR | $PO) $CM* ($ALPlus | $HL); 314($ALPlus | $HL) $CM* ($PR | $PO); 315^$CM+ ($PR | $PO); # Rule 10, any otherwise unattached CM behaves as AL 316 317# 318# LB 25 Numbers. 319# 320(($PR | $PO) $CM*)? (($OP | $HY) $CM*)? ($IS $CM*)? $NU ($CM* ($NU | $SY | $IS))* 321 ($CM* ($CL | $CP))? ($CM* ($PR | $PO))?; 322 323# LB 26 Do not break a Korean syllable 324# 325$JL $CM* ($JL | $JV | $H2 | $H3); 326($JV | $H2) $CM* ($JV | $JT); 327($JT | $H3) $CM* $JT; 328 329# LB 27 Treat korean Syllable Block the same as ID (don't break it) 330($JL | $JV | $JT | $H2 | $H3) $CM* $IN; 331($JL | $JV | $JT | $H2 | $H3) $CM* $PO; 332$PR $CM* ($JL | $JV | $JT | $H2 | $H3); 333 334 335# LB 28 Do not break between alphabetics 336# 337($ALPlus | $HL) $CM* ($ALPlus | $HL); 338^$CM+ ($ALPlus | $HL); # The $CM+ is from rule 10, an unattached CM is treated as AL 339 340# LB 29 341$IS $CM* ($ALPlus | $HL); 342 343# LB 30 344($ALPlus | $HL | $NU) $CM* $OP30; 345^$CM+ $OP30; # The $CM+ is from rule 10, an unattached CM is treated as AL. 346$CP30 $CM* ($ALPlus | $HL | $NU); 347 348# LB 30a Do not break between regional indicators. Break after pairs of them. 349# Tricky interaction with LB8a: ZWJ x . together with ZWJ acting like a CM. 350$RI $CM* $RI / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $CM]]; 351$RI $CM* $RI $CM* [$CM-$ZWJ] / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $CM]]; 352$RI $CM* $RI $CM* [$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $ZWJ {eof}]; 353# note: the preceding rule includes {eof} rather than having the last [set] term qualified with '?' 354# because of the chain-out behavior difference. The rule must chain out only from the [set characters], 355# not from the preceding $RI or $CM, which it would be able to do if the set were optional. 356 357# LB 30b Do not break between an Emoji Base and an Emoji Modifier 358$EB $CM* $EM; 359 360# LB 31 Break everywhere else. 361# Match a single code point if no other rule applies. 362.; 363