Skip to content

G
Geth-Modification
Commit 5bee5d69 authored by Elad's avatar Elad Committed by Balint Gabor
Browse files
Options

vendor: added vendor packages necessary for the swarm-network-rewrite merge (#16792) 

* vendor: added vendor packages necessary for the swarm-network-rewrite merge into ethereum master

* vendor: removed multihash deps
parent cbfb40b0
Show whitespace changes
Inline Side-by-side
Showing with 29663 additions and 623 deletions
vendor/github.com/naoina/toml/encode.go
View file @ 5bee5d69
......@@ -61,20 +61,26 @@ func (cfg *Config) NewEncoder(w io.Writer) *Encoder {
// Encode writes the TOML of v to the stream.
// See the documentation for Marshal for details about the conversion of Go values to TOML.
func (e *Encoder) Encode(v interface{}) error {
rv := reflect.ValueOf(v)
var (
buf = &tableBuf{typ: ast.TableTypeNormal}
rv = reflect.ValueOf(v)
err error
)
for rv.Kind() == reflect.Ptr {
if rv.IsNil() {
return &marshalNilError{rv.Type()}
}
rv = rv.Elem()
}
buf := &tableBuf{typ: ast.TableTypeNormal}
var err error
switch rv.Kind() {
case reflect.Struct:
err = buf.structFields(e.cfg, rv)
case reflect.Map:
err = buf.mapFields(e.cfg, rv)
case reflect.Interface:
return e.Encode(rv.Interface())
default:
err = &marshalTableError{rv.Type()}
}
......
vendor/github.com/naoina/toml/parse.go
View file @ 5bee5d69
......@@ -97,6 +97,7 @@ type toml struct {
currentTable *ast.Table
s string
key string
tableKeys []string
val ast.Value
arr *array
stack []*stack
......@@ -180,12 +181,12 @@ func (p *tomlParser) SetArray(begin, end int) {
}
func (p *toml) SetTable(buf []rune, begin, end int) {
p.setTable(p.table, buf, begin, end)
rawName := string(buf[begin:end])
p.setTable(p.table, rawName, p.tableKeys)
p.tableKeys = nil
}
func (p *toml) setTable(parent *ast.Table, buf []rune, begin, end int) {
name := string(buf[begin:end])
names := splitTableKey(name)
func (p *toml) setTable(parent *ast.Table, name string, names []string) {
parent, err := p.lookupTable(parent, names[:len(names)-1])
if err != nil {
p.Error(err)
......@@ -230,12 +231,12 @@ func (p *tomlParser) SetTableString(begin, end int) {
}
func (p *toml) SetArrayTable(buf []rune, begin, end int) {
p.setArrayTable(p.table, buf, begin, end)
rawName := string(buf[begin:end])
p.setArrayTable(p.table, rawName, p.tableKeys)
p.tableKeys = nil
}
func (p *toml) setArrayTable(parent *ast.Table, buf []rune, begin, end int) {
name := string(buf[begin:end])
names := splitTableKey(name)
func (p *toml) setArrayTable(parent *ast.Table, name string, names []string) {
parent, err := p.lookupTable(parent, names[:len(names)-1])
if err != nil {
p.Error(err)
......@@ -260,11 +261,11 @@ func (p *toml) setArrayTable(parent *ast.Table, buf []rune, begin, end int) {
func (p *toml) StartInlineTable() {
p.skip = false
p.stack = append(p.stack, &stack{p.key, p.currentTable})
buf := []rune(p.key)
names := []string{p.key}
if p.arr == nil {
p.setTable(p.currentTable, buf, 0, len(buf))
p.setTable(p.currentTable, names[0], names)
} else {
p.setArrayTable(p.currentTable, buf, 0, len(buf))
p.setArrayTable(p.currentTable, names[0], names)
}
}
......@@ -282,6 +283,13 @@ func (p *toml) AddLineCount(i int) {
func (p *toml) SetKey(buf []rune, begin, end int) {
p.key = string(buf[begin:end])
if len(p.key) > 0 && p.key[0] == '"' {
p.key = p.unquote(p.key)
}
}
func (p *toml) AddTableKey() {
p.tableKeys = append(p.tableKeys, p.key)
}
func (p *toml) AddKeyValue() {
......@@ -352,25 +360,3 @@ func (p *toml) lookupTable(t *ast.Table, keys []string) (*ast.Table, error) {
}
return t, nil
}
func splitTableKey(tk string) []string {
key := make([]byte, 0, 1)
keys := make([]string, 0, 1)
inQuote := false
for i := 0; i < len(tk); i++ {
k := tk[i]
switch {
case k == tableSeparator && !inQuote:
keys = append(keys, string(key))
key = key[:0] // reuse buffer.
case k == '"':
inQuote = !inQuote
case (k == ' ' || k == '\t') && !inQuote:
// skip.
default:
key = append(key, k)
}
}
keys = append(keys, string(key))
return keys
}
vendor/github.com/naoina/toml/parse.peg
View file @ 5bee5d69
......@@ -29,7 +29,7 @@ key <- bareKey / quotedKey
bareKey <- <[0-9A-Za-z\-_]+> { p.SetKey(p.buffer, begin, end) }
quotedKey <- '"' <basicChar+> '"' { p.SetKey(p.buffer, begin-1, end+1) }
quotedKey <- < '"' basicChar* '"' > { p.SetKey(p.buffer, begin, end) }
val <- (
<datetime> { p.SetTime(begin, end) }
......@@ -55,7 +55,9 @@ inlineTable <- (
inlineTableKeyValues <- (keyval inlineTableValSep?)*
tableKey <- key (tableKeySep key)*
tableKey <- tableKeyComp (tableKeySep tableKeyComp)*
tableKeyComp <- key { p.AddTableKey() }
tableKeySep <- ws '.' ws
......@@ -117,7 +119,7 @@ timeNumoffset <- [\-+] timeHour ':' timeMinute
timeOffset <- 'Z' / timeNumoffset
partialTime <- timeHour ':' timeMinute ':' timeSecond timeSecfrac?
fullDate <- dateFullYear '-' dateMonth '-' dateMDay
fullTime <- partialTime timeOffset
fullTime <- partialTime timeOffset?
datetime <- (fullDate ('T' fullTime)?) / partialTime
digit <- [0-9]
......
vendor/github.com/naoina/toml/parse.peg.go
View file @ 5bee5d69
......@@ -31,6 +31,7 @@ const (
ruleinlineTable
ruleinlineTableKeyValues
ruletableKey
ruletableKeyComp
ruletableKeySep
ruleinlineTableValSep
ruleinteger
......@@ -99,6 +100,7 @@ const (
ruleAction22
ruleAction23
ruleAction24
ruleAction25
)
var rul3s = [...]string{
......@@ -120,6 +122,7 @@ var rul3s = [...]string{
"inlineTable",
"inlineTableKeyValues",
"tableKey",
"tableKeyComp",
"tableKeySep",
"inlineTableValSep",
"integer",
......@@ -188,6 +191,7 @@ var rul3s = [...]string{
"Action22",
"Action23",
"Action24",
"Action25",
}
type token32 struct {
......@@ -304,7 +308,7 @@ type tomlParser struct {
Buffer string
buffer []rune
rules [86]func() bool
rules [88]func() bool
parse func(rule ...int) error
reset func()
Pretty bool
......@@ -409,7 +413,7 @@ func (p *tomlParser) Execute() {
case ruleAction5:
p.SetKey(p.buffer, begin, end)
case ruleAction6:
p.SetKey(p.buffer, begin-1, end+1)
p.SetKey(p.buffer, begin, end)
case ruleAction7:
p.SetTime(begin, end)
case ruleAction8:
......@@ -431,21 +435,23 @@ func (p *tomlParser) Execute() {
case ruleAction16:
p.EndInlineTable()
case ruleAction17:
p.SetBasicString(p.buffer, begin, end)
p.AddTableKey()
case ruleAction18:
p.SetMultilineString()
p.SetBasicString(p.buffer, begin, end)
case ruleAction19:
p.AddMultilineBasicBody(p.buffer, begin, end)
p.SetMultilineString()
case ruleAction20:
p.SetLiteralString(p.buffer, begin, end)
p.AddMultilineBasicBody(p.buffer, begin, end)
case ruleAction21:
p.SetMultilineLiteralString(p.buffer, begin, end)
p.SetLiteralString(p.buffer, begin, end)
case ruleAction22:
p.StartArray()
p.SetMultilineLiteralString(p.buffer, begin, end)
case ruleAction23:
p.AddArrayVal()
p.StartArray()
case ruleAction24:
p.AddArrayVal()
case ruleAction25:
p.AddArrayVal()
}
}
......@@ -1069,15 +1075,12 @@ func (p *tomlParser) Init() {
position, tokenIndex = position72, tokenIndex72
{
position81 := position
if buffer[position] != rune('"') {
goto l70
}
position++
{
position82 := position
if !_rules[rulebasicChar]() {
if buffer[position] != rune('"') {
goto l70
}
position++
l83:
{
position84, tokenIndex84 := position, tokenIndex
......@@ -1088,12 +1091,12 @@ func (p *tomlParser) Init() {
l84:
position, tokenIndex = position84, tokenIndex84
}
add(rulePegText, position82)
}
if buffer[position] != rune('"') {
goto l70
}
position++
add(rulePegText, position82)
}
{
add(ruleAction6, position)
}
......@@ -1110,7 +1113,7 @@ func (p *tomlParser) Init() {
},
/* 8 bareKey <- <(<((&('_') '_') | (&('-') '-') | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]))+> Action5)> */
nil,
/* 9 quotedKey <- <('"' <basicChar+> '"' Action6)> */
/* 9 quotedKey <- <(<('"' basicChar* '"')> Action6)> */
nil,
/* 10 val <- <((<datetime> Action7) / (<float> Action8) / ((&('{') inlineTable) | (&('[') (<array> Action12)) | (&('f' | 't') (<boolean> Action11)) | (&('"' | '\'') (<string> Action10)) | (&('+' | '-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') (<integer> Action9))))> */
func() bool {
......@@ -1177,49 +1180,56 @@ func (p *tomlParser) Init() {
goto l101
}
{
position104 := position
position104, tokenIndex104 := position, tokenIndex
{
position106 := position
{
position105, tokenIndex105 := position, tokenIndex
position107, tokenIndex107 := position, tokenIndex
if buffer[position] != rune('Z') {
goto l106
goto l108
}
position++
goto l105
l106:
position, tokenIndex = position105, tokenIndex105
goto l107
l108:
position, tokenIndex = position107, tokenIndex107
{
position107 := position
position109 := position
{
position108, tokenIndex108 := position, tokenIndex
position110, tokenIndex110 := position, tokenIndex
if buffer[position] != rune('-') {
goto l109
goto l111
}
position++
goto l108
l109:
position, tokenIndex = position108, tokenIndex108
goto l110
l111:
position, tokenIndex = position110, tokenIndex110
if buffer[position] != rune('+') {
goto l101
goto l104
}
position++
}
l108:
l110:
if !_rules[ruletimeHour]() {
goto l101
goto l104
}
if buffer[position] != rune(':') {
goto l101
goto l104
}
position++
if !_rules[ruletimeMinute]() {
goto l101
goto l104
}
add(ruletimeNumoffset, position107)
add(ruletimeNumoffset, position109)
}
}
l105:
add(ruletimeOffset, position104)
l107:
add(ruletimeOffset, position106)
}
goto l105
l104:
position, tokenIndex = position104, tokenIndex104
}
l105:
add(rulefullTime, position103)
}
goto l102
......@@ -1246,33 +1256,20 @@ func (p *tomlParser) Init() {
l91:
position, tokenIndex = position90, tokenIndex90
{
position112 := position
position114 := position
{
position113 := position
position115 := position
if !_rules[ruleinteger]() {
goto l111
}
{
position114, tokenIndex114 := position, tokenIndex
if !_rules[rulefrac]() {
goto l115
goto l113
}
{
position116, tokenIndex116 := position, tokenIndex
if !_rules[ruleexp]() {
goto l116
}
if !_rules[rulefrac]() {
goto l117
l116:
position, tokenIndex = position116, tokenIndex116
}
l117:
goto l114
l115:
position, tokenIndex = position114, tokenIndex114
{
position118, tokenIndex118 := position, tokenIndex
if !_rules[rulefrac]() {
if !_rules[ruleexp]() {
goto l118
}
goto l119
......@@ -1280,26 +1277,39 @@ func (p *tomlParser) Init() {
position, tokenIndex = position118, tokenIndex118
}
l119:
goto l116
l117:
position, tokenIndex = position116, tokenIndex116
{
position120, tokenIndex120 := position, tokenIndex
if !_rules[rulefrac]() {
goto l120
}
goto l121
l120:
position, tokenIndex = position120, tokenIndex120
}
l121:
if !_rules[ruleexp]() {
goto l111
goto l113
}
}
l114:
add(rulefloat, position113)
l116:
add(rulefloat, position115)
}
add(rulePegText, position112)
add(rulePegText, position114)
}
{
add(ruleAction8, position)
}
goto l90
l111:
l113:
position, tokenIndex = position90, tokenIndex90
{
switch buffer[position] {
case '{':
{
position122 := position
position124 := position
if buffer[position] != rune('{') {
goto l88
}
......@@ -1311,39 +1321,39 @@ func (p *tomlParser) Init() {
goto l88
}
{
position124 := position
l125:
position126 := position
l127:
{
position126, tokenIndex126 := position, tokenIndex
position128, tokenIndex128 := position, tokenIndex
if !_rules[rulekeyval]() {
goto l126
goto l128
}
{
position127, tokenIndex127 := position, tokenIndex
position129, tokenIndex129 := position, tokenIndex
{
position129 := position
position131 := position
if !_rules[rulews]() {
goto l127
goto l129
}
if buffer[position] != rune(',') {
goto l127
goto l129
}
position++
if !_rules[rulews]() {
goto l127
goto l129
}
add(ruleinlineTableValSep, position129)
add(ruleinlineTableValSep, position131)
}
goto l128
l127:
position, tokenIndex = position127, tokenIndex127
goto l130
l129:
position, tokenIndex = position129, tokenIndex129
}
l130:
goto l127
l128:
goto l125
l126:
position, tokenIndex = position126, tokenIndex126
position, tokenIndex = position128, tokenIndex128
}
add(ruleinlineTableKeyValues, position124)
add(ruleinlineTableKeyValues, position126)
}
if !_rules[rulews]() {
goto l88
......@@ -1355,58 +1365,39 @@ func (p *tomlParser) Init() {
{
add(ruleAction16, position)
}
add(ruleinlineTable, position122)
add(ruleinlineTable, position124)
}
break
case '[':
{
position131 := position
position133 := position
{
position132 := position
position134 := position
if buffer[position] != rune('[') {
goto l88
}
position++
{
add(ruleAction22, position)
add(ruleAction23, position)
}
if !_rules[rulewsnl]() {
goto l88
}
{
position134, tokenIndex134 := position, tokenIndex
position136, tokenIndex136 := position, tokenIndex
{
position136 := position
position138 := position
if !_rules[ruleval]() {
goto l134
goto l136
}
{
add(ruleAction23, position)
add(ruleAction24, position)
}
l138:
l140:
{
position139, tokenIndex139 := position, tokenIndex
position141, tokenIndex141 := position, tokenIndex
if !_rules[rulewsnl]() {
goto l139
}
{
position140, tokenIndex140 := position, tokenIndex
if !_rules[rulecomment]() {
goto l140
}
goto l141
l140:
position, tokenIndex = position140, tokenIndex140
}
l141:
if !_rules[rulewsnl]() {
goto l139
}
if !_rules[rulearraySep]() {
goto l139
}
if !_rules[rulewsnl]() {
goto l139
}
{
position142, tokenIndex142 := position, tokenIndex
......@@ -1419,37 +1410,43 @@ func (p *tomlParser) Init() {
}
l143:
if !_rules[rulewsnl]() {
goto l139
goto l141
}
if !_rules[ruleval]() {
goto l139
if !_rules[rulearraySep]() {
goto l141
}
if !_rules[rulewsnl]() {
goto l141
}
{
add(ruleAction24, position)
position144, tokenIndex144 := position, tokenIndex
if !_rules[rulecomment]() {
goto l144
}
goto l138
l139:
position, tokenIndex = position139, tokenIndex139
goto l145
l144:
position, tokenIndex = position144, tokenIndex144
}
l145:
if !_rules[rulewsnl]() {
goto l134
goto l141
}
if !_rules[ruleval]() {
goto l141
}
{
position145, tokenIndex145 := position, tokenIndex
if !_rules[rulearraySep]() {
goto l145
add(ruleAction25, position)
}
goto l146
l145:
position, tokenIndex = position145, tokenIndex145
goto l140
l141:
position, tokenIndex = position141, tokenIndex141
}
l146:
if !_rules[rulewsnl]() {
goto l134
goto l136
}
{
position147, tokenIndex147 := position, tokenIndex
if !_rules[rulecomment]() {
if !_rules[rulearraySep]() {
goto l147
}
goto l148
......@@ -1457,13 +1454,26 @@ func (p *tomlParser) Init() {
position, tokenIndex = position147, tokenIndex147
}
l148:
add(rulearrayValues, position136)
if !_rules[rulewsnl]() {
goto l136
}
{
position149, tokenIndex149 := position, tokenIndex
if !_rules[rulecomment]() {
goto l149
}
goto l135
l134:
position, tokenIndex = position134, tokenIndex134
goto l150
l149:
position, tokenIndex = position149, tokenIndex149
}
l135:
l150:
add(rulearrayValues, position138)
}
goto l137
l136:
position, tokenIndex = position136, tokenIndex136
}
l137:
if !_rules[rulewsnl]() {
goto l88
}
......@@ -1471,9 +1481,9 @@ func (p *tomlParser) Init() {
goto l88
}
position++
add(rulearray, position132)
add(rulearray, position134)
}
add(rulePegText, position131)
add(rulePegText, position133)
}
{
add(ruleAction12, position)
......@@ -1481,30 +1491,30 @@ func (p *tomlParser) Init() {
break
case 'f', 't':
{
position150 := position
position152 := position
{
position151 := position
position153 := position
{
position152, tokenIndex152 := position, tokenIndex
position154, tokenIndex154 := position, tokenIndex
if buffer[position] != rune('t') {
goto l153
goto l155
}
position++
if buffer[position] != rune('r') {
goto l153
goto l155
}
position++
if buffer[position] != rune('u') {
goto l153
goto l155
}
position++
if buffer[position] != rune('e') {
goto l153
goto l155
}
position++
goto l152
l153:
position, tokenIndex = position152, tokenIndex152
goto l154
l155:
position, tokenIndex = position154, tokenIndex154
if buffer[position] != rune('f') {
goto l88
}
......@@ -1526,10 +1536,10 @@ func (p *tomlParser) Init() {
}
position++
}
l152:
add(ruleboolean, position151)
l154:
add(ruleboolean, position153)
}
add(rulePegText, position150)
add(rulePegText, position152)
}
{
add(ruleAction11, position)
......@@ -1537,278 +1547,278 @@ func (p *tomlParser) Init() {
break
case '"', '\'':
{
position155 := position
position157 := position
{
position156 := position
position158 := position
{
position157, tokenIndex157 := position, tokenIndex
position159, tokenIndex159 := position, tokenIndex
{
position159 := position
position161 := position
if buffer[position] != rune('\'') {
goto l158
goto l160
}
position++
if buffer[position] != rune('\'') {
goto l158
goto l160
}
position++
if buffer[position] != rune('\'') {
goto l158
goto l160
}
position++
{
position160 := position
position162 := position
{
position161 := position
l162:
position163 := position
l164:
{
position163, tokenIndex163 := position, tokenIndex
position165, tokenIndex165 := position, tokenIndex
{
position164, tokenIndex164 := position, tokenIndex
position166, tokenIndex166 := position, tokenIndex
if buffer[position] != rune('\'') {
goto l164
goto l166
}
position++
if buffer[position] != rune('\'') {
goto l164
goto l166
}
position++
if buffer[position] != rune('\'') {
goto l164
goto l166
}
position++
goto l163
l164:
position, tokenIndex = position164, tokenIndex164
goto l165
l166:
position, tokenIndex = position166, tokenIndex166
}
{
position165, tokenIndex165 := position, tokenIndex
position167, tokenIndex167 := position, tokenIndex
{
position167 := position
position169 := position
{
position168, tokenIndex168 := position, tokenIndex
position170, tokenIndex170 := position, tokenIndex
if buffer[position] != rune('\t') {
goto l169
goto l171
}
position++
goto l168
l169:
position, tokenIndex = position168, tokenIndex168
goto l170
l171:
position, tokenIndex = position170, tokenIndex170
if c := buffer[position]; c < rune(' ') || c > rune('\U0010ffff') {
goto l166
goto l168
}
position++
}
l168:
add(rulemlLiteralChar, position167)
l170:
add(rulemlLiteralChar, position169)
}
goto l165
l166:
position, tokenIndex = position165, tokenIndex165
goto l167
l168:
position, tokenIndex = position167, tokenIndex167
if !_rules[rulenewline]() {
goto l163
goto l165
}
}
l167:
goto l164
l165:
goto l162
l163:
position, tokenIndex = position163, tokenIndex163
position, tokenIndex = position165, tokenIndex165
}
add(rulemlLiteralBody, position161)
add(rulemlLiteralBody, position163)
}
add(rulePegText, position160)
add(rulePegText, position162)
}
if buffer[position] != rune('\'') {
goto l158
goto l160
}
position++
if buffer[position] != rune('\'') {
goto l158
goto l160
}
position++
if buffer[position] != rune('\'') {
goto l158
goto l160
}
position++
{
add(ruleAction21, position)
add(ruleAction22, position)
}
add(rulemlLiteralString, position159)
add(rulemlLiteralString, position161)
}
goto l157
l158:
position, tokenIndex = position157, tokenIndex157
goto l159
l160:
position, tokenIndex = position159, tokenIndex159
{
position172 := position
position174 := position
if buffer[position] != rune('\'') {
goto l171
goto l173
}
position++
{
position173 := position
l174:
position175 := position
l176:
{
position175, tokenIndex175 := position, tokenIndex
position177, tokenIndex177 := position, tokenIndex
{
position176 := position
position178 := position
{
switch buffer[position] {
case '\t':
if buffer[position] != rune('\t') {
goto l175
goto l177
}
position++
break
case ' ', '!', '"', '#', '$', '%', '&':
if c := buffer[position]; c < rune(' ') || c > rune('&') {
goto l175
goto l177
}
position++
break
default:
if c := buffer[position]; c < rune('(') || c > rune('\U0010ffff') {
goto l175
goto l177
}
position++
break
}
}
add(ruleliteralChar, position176)
add(ruleliteralChar, position178)
}
goto l174
l175:
position, tokenIndex = position175, tokenIndex175
goto l176
l177:
position, tokenIndex = position177, tokenIndex177
}
add(rulePegText, position173)
add(rulePegText, position175)
}
if buffer[position] != rune('\'') {
goto l171
goto l173
}
position++
{
add(ruleAction20, position)
add(ruleAction21, position)
}
add(ruleliteralString, position172)
add(ruleliteralString, position174)
}
goto l157
l171:
position, tokenIndex = position157, tokenIndex157
goto l159
l173:
position, tokenIndex = position159, tokenIndex159
{
position180 := position
position182 := position
if buffer[position] != rune('"') {
goto l179
goto l181
}
position++
if buffer[position] != rune('"') {
goto l179
goto l181
}
position++
if buffer[position] != rune('"') {
goto l179
goto l181
}
position++
{
position181 := position
l182:
position183 := position
l184:
{
position183, tokenIndex183 := position, tokenIndex
position185, tokenIndex185 := position, tokenIndex
{
position184, tokenIndex184 := position, tokenIndex
position186, tokenIndex186 := position, tokenIndex
{
position186 := position
position188 := position
{
position187, tokenIndex187 := position, tokenIndex
position189, tokenIndex189 := position, tokenIndex
if !_rules[rulebasicChar]() {
goto l188
goto l190
}
goto l187
l188:
position, tokenIndex = position187, tokenIndex187
goto l189
l190:
position, tokenIndex = position189, tokenIndex189
if !_rules[rulenewline]() {
goto l185
goto l187
}
}
l187:
add(rulePegText, position186)
l189:
add(rulePegText, position188)
}
{
add(ruleAction19, position)
add(ruleAction20, position)
}
goto l184
l185:
position, tokenIndex = position184, tokenIndex184
goto l186
l187:
position, tokenIndex = position186, tokenIndex186
if !_rules[ruleescape]() {
goto l183
goto l185
}
if !_rules[rulenewline]() {
goto l183
goto l185
}
if !_rules[rulewsnl]() {
goto l183
goto l185
}
}
l184:
goto l182
l183:
position, tokenIndex = position183, tokenIndex183
l186:
goto l184
l185:
position, tokenIndex = position185, tokenIndex185
}
add(rulemlBasicBody, position181)
add(rulemlBasicBody, position183)
}
if buffer[position] != rune('"') {
goto l179
goto l181
}
position++
if buffer[position] != rune('"') {
goto l179
goto l181
}
position++
if buffer[position] != rune('"') {
goto l179
goto l181
}
position++
{
add(ruleAction18, position)
add(ruleAction19, position)
}
add(rulemlBasicString, position180)
add(rulemlBasicString, position182)
}
goto l157
l179:
position, tokenIndex = position157, tokenIndex157
goto l159
l181:
position, tokenIndex = position159, tokenIndex159
{
position191 := position
position193 := position
{
position192 := position
position194 := position
if buffer[position] != rune('"') {
goto l88
}
position++
l193:
l195:
{
position194, tokenIndex194 := position, tokenIndex
position196, tokenIndex196 := position, tokenIndex
if !_rules[rulebasicChar]() {
goto l194
goto l196
}
goto l193
l194:
position, tokenIndex = position194, tokenIndex194
goto l195
l196:
position, tokenIndex = position196, tokenIndex196
}
if buffer[position] != rune('"') {
goto l88
}
position++
add(rulePegText, position192)
add(rulePegText, position194)
}
{
add(ruleAction17, position)
add(ruleAction18, position)
}
add(rulebasicString, position191)
add(rulebasicString, position193)
}
}
l157:
add(rulestring, position156)
l159:
add(rulestring, position158)
}
add(rulePegText, position155)
add(rulePegText, position157)
}
{
add(ruleAction10, position)
......@@ -1816,11 +1826,11 @@ func (p *tomlParser) Init() {
break
default:
{
position197 := position
position199 := position
if !_rules[ruleinteger]() {
goto l88
}
add(rulePegText, position197)
add(rulePegText, position199)
}
{
add(ruleAction9, position)
......@@ -1848,708 +1858,728 @@ func (p *tomlParser) Init() {
nil,
/* 15 inlineTableKeyValues <- <(keyval inlineTableValSep?)*> */
nil,
/* 16 tableKey <- <(key (tableKeySep key)*)> */
/* 16 tableKey <- <(tableKeyComp (tableKeySep tableKeyComp)*)> */
func() bool {
position204, tokenIndex204 := position, tokenIndex
position206, tokenIndex206 := position, tokenIndex
{
position205 := position
if !_rules[rulekey]() {
goto l204
position207 := position
if !_rules[ruletableKeyComp]() {
goto l206
}
l206:
l208:
{
position207, tokenIndex207 := position, tokenIndex
position209, tokenIndex209 := position, tokenIndex
{
position208 := position
position210 := position
if !_rules[rulews]() {
goto l207
goto l209
}
if buffer[position] != rune('.') {
goto l207
goto l209
}
position++
if !_rules[rulews]() {
goto l207
goto l209
}
add(ruletableKeySep, position210)
}
if !_rules[ruletableKeyComp]() {
goto l209
}
add(ruletableKeySep, position208)
goto l208
l209:
position, tokenIndex = position209, tokenIndex209
}
add(ruletableKey, position207)
}
return true
l206:
position, tokenIndex = position206, tokenIndex206
return false
},
/* 17 tableKeyComp <- <(key Action17)> */
func() bool {
position211, tokenIndex211 := position, tokenIndex
{
position212 := position
if !_rules[rulekey]() {
goto l207
goto l211
}
goto l206
l207:
position, tokenIndex = position207, tokenIndex207
{
add(ruleAction17, position)
}
add(ruletableKey, position205)
add(ruletableKeyComp, position212)
}
return true
l204:
position, tokenIndex = position204, tokenIndex204
l211:
position, tokenIndex = position211, tokenIndex211
return false
},
/* 17 tableKeySep <- <(ws '.' ws)> */
/* 18 tableKeySep <- <(ws '.' ws)> */
nil,
/* 18 inlineTableValSep <- <(ws ',' ws)> */
/* 19 inlineTableValSep <- <(ws ',' ws)> */
nil,
/* 19 integer <- <(('-' / '+')? int)> */
/* 20 integer <- <(('-' / '+')? int)> */
func() bool {
position211, tokenIndex211 := position, tokenIndex
position216, tokenIndex216 := position, tokenIndex
{
position212 := position
position217 := position
{
position213, tokenIndex213 := position, tokenIndex
position218, tokenIndex218 := position, tokenIndex
{
position215, tokenIndex215 := position, tokenIndex
position220, tokenIndex220 := position, tokenIndex
if buffer[position] != rune('-') {
goto l216
goto l221
}
position++
goto l215
l216:
position, tokenIndex = position215, tokenIndex215
goto l220
l221:
position, tokenIndex = position220, tokenIndex220
if buffer[position] != rune('+') {
goto l213
goto l218
}
position++
}
l215:
goto l214
l213:
position, tokenIndex = position213, tokenIndex213
l220:
goto l219
l218:
position, tokenIndex = position218, tokenIndex218
}
l214:
l219:
{
position217 := position
position222 := position
{
position218, tokenIndex218 := position, tokenIndex
position223, tokenIndex223 := position, tokenIndex
if c := buffer[position]; c < rune('1') || c > rune('9') {
goto l219
goto l224
}
position++
{
position222, tokenIndex222 := position, tokenIndex
position227, tokenIndex227 := position, tokenIndex
if !_rules[ruledigit]() {
goto l223
goto l228
}
goto l222
l223:
position, tokenIndex = position222, tokenIndex222
goto l227
l228:
position, tokenIndex = position227, tokenIndex227
if buffer[position] != rune('_') {
goto l219
goto l224
}
position++
if !_rules[ruledigit]() {
goto l219
goto l224
}
}
l222:
l220:
l227:
l225:
{
position221, tokenIndex221 := position, tokenIndex
position226, tokenIndex226 := position, tokenIndex
{
position224, tokenIndex224 := position, tokenIndex
position229, tokenIndex229 := position, tokenIndex
if !_rules[ruledigit]() {
goto l225
goto l230
}
goto l224
l225:
position, tokenIndex = position224, tokenIndex224
goto l229
l230:
position, tokenIndex = position229, tokenIndex229
if buffer[position] != rune('_') {
goto l221
goto l226
}
position++
if !_rules[ruledigit]() {
goto l221
goto l226
}
}
l224:
goto l220
l221:
position, tokenIndex = position221, tokenIndex221
l229:
goto l225
l226:
position, tokenIndex = position226, tokenIndex226
}
goto l218
l219:
position, tokenIndex = position218, tokenIndex218
goto l223
l224:
position, tokenIndex = position223, tokenIndex223
if !_rules[ruledigit]() {
goto l211
goto l216
}
}
l218:
add(ruleint, position217)
l223:
add(ruleint, position222)
}
add(ruleinteger, position212)
add(ruleinteger, position217)
}
return true
l211:
position, tokenIndex = position211, tokenIndex211
l216:
position, tokenIndex = position216, tokenIndex216
return false
},
/* 20 int <- <(([1-9] (digit / ('_' digit))+) / digit)> */
/* 21 int <- <(([1-9] (digit / ('_' digit))+) / digit)> */
nil,
/* 21 float <- <(integer ((frac exp?) / (frac? exp)))> */
/* 22 float <- <(integer ((frac exp?) / (frac? exp)))> */
nil,
/* 22 frac <- <('.' digit (digit / ('_' digit))*)> */
/* 23 frac <- <('.' digit (digit / ('_' digit))*)> */
func() bool {
position228, tokenIndex228 := position, tokenIndex
position233, tokenIndex233 := position, tokenIndex
{
position229 := position
position234 := position
if buffer[position] != rune('.') {
goto l228
goto l233
}
position++
if !_rules[ruledigit]() {
goto l228
goto l233
}
l230:
l235:
{
position231, tokenIndex231 := position, tokenIndex
position236, tokenIndex236 := position, tokenIndex
{
position232, tokenIndex232 := position, tokenIndex
position237, tokenIndex237 := position, tokenIndex
if !_rules[ruledigit]() {
goto l233
goto l238
}
goto l232
l233:
position, tokenIndex = position232, tokenIndex232
goto l237
l238:
position, tokenIndex = position237, tokenIndex237
if buffer[position] != rune('_') {
goto l231
goto l236
}
position++
if !_rules[ruledigit]() {
goto l231
goto l236
}
}
l232:
goto l230
l231:
position, tokenIndex = position231, tokenIndex231
l237:
goto l235
l236:
position, tokenIndex = position236, tokenIndex236
}
add(rulefrac, position229)
add(rulefrac, position234)
}
return true
l228:
position, tokenIndex = position228, tokenIndex228
l233:
position, tokenIndex = position233, tokenIndex233
return false
},
/* 23 exp <- <(('e' / 'E') ('-' / '+')? digit (digit / ('_' digit))*)> */
/* 24 exp <- <(('e' / 'E') ('-' / '+')? digit (digit / ('_' digit))*)> */
func() bool {
position234, tokenIndex234 := position, tokenIndex
position239, tokenIndex239 := position, tokenIndex
{
position235 := position
position240 := position
{
position236, tokenIndex236 := position, tokenIndex
position241, tokenIndex241 := position, tokenIndex
if buffer[position] != rune('e') {
goto l237
goto l242
}
position++
goto l236
l237:
position, tokenIndex = position236, tokenIndex236
goto l241
l242:
position, tokenIndex = position241, tokenIndex241
if buffer[position] != rune('E') {
goto l234
goto l239
}
position++
}
l236:
l241:
{
position238, tokenIndex238 := position, tokenIndex
position243, tokenIndex243 := position, tokenIndex
{
position240, tokenIndex240 := position, tokenIndex
position245, tokenIndex245 := position, tokenIndex
if buffer[position] != rune('-') {
goto l241
goto l246
}
position++
goto l240
l241:
position, tokenIndex = position240, tokenIndex240
goto l245
l246:
position, tokenIndex = position245, tokenIndex245
if buffer[position] != rune('+') {
goto l238
goto l243
}
position++
}
l240:
goto l239
l238:
position, tokenIndex = position238, tokenIndex238
l245:
goto l244
l243:
position, tokenIndex = position243, tokenIndex243
}
l239:
l244:
if !_rules[ruledigit]() {
goto l234
goto l239
}
l242:
l247:
{
position243, tokenIndex243 := position, tokenIndex
position248, tokenIndex248 := position, tokenIndex
{
position244, tokenIndex244 := position, tokenIndex
position249, tokenIndex249 := position, tokenIndex
if !_rules[ruledigit]() {
goto l245
goto l250
}
goto l244
l245:
position, tokenIndex = position244, tokenIndex244
goto l249
l250:
position, tokenIndex = position249, tokenIndex249
if buffer[position] != rune('_') {
goto l243
goto l248
}
position++
if !_rules[ruledigit]() {
goto l243
goto l248
}
}
l244:
goto l242
l243:
position, tokenIndex = position243, tokenIndex243
l249:
goto l247
l248:
position, tokenIndex = position248, tokenIndex248
}
add(ruleexp, position235)
add(ruleexp, position240)
}
return true
l234:
position, tokenIndex = position234, tokenIndex234
l239:
position, tokenIndex = position239, tokenIndex239
return false
},
/* 24 string <- <(mlLiteralString / literalString / mlBasicString / basicString)> */
/* 25 string <- <(mlLiteralString / literalString / mlBasicString / basicString)> */
nil,
/* 25 basicString <- <(<('"' basicChar* '"')> Action17)> */
/* 26 basicString <- <(<('"' basicChar* '"')> Action18)> */
nil,
/* 26 basicChar <- <(basicUnescaped / escaped)> */
/* 27 basicChar <- <(basicUnescaped / escaped)> */
func() bool {
position248, tokenIndex248 := position, tokenIndex
position253, tokenIndex253 := position, tokenIndex
{
position249 := position
position254 := position
{
position250, tokenIndex250 := position, tokenIndex
position255, tokenIndex255 := position, tokenIndex
{
position252 := position
position257 := position
{
switch buffer[position] {
case ' ', '!':
if c := buffer[position]; c < rune(' ') || c > rune('!') {
goto l251
goto l256
}
position++
break
case '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[':
if c := buffer[position]; c < rune('#') || c > rune('[') {
goto l251
goto l256
}
position++
break
default:
if c := buffer[position]; c < rune(']') || c > rune('\U0010ffff') {
goto l251
goto l256
}
position++
break
}
}
add(rulebasicUnescaped, position252)
add(rulebasicUnescaped, position257)
}
goto l250
l251:
position, tokenIndex = position250, tokenIndex250
goto l255
l256:
position, tokenIndex = position255, tokenIndex255
{
position254 := position
position259 := position
if !_rules[ruleescape]() {
goto l248
goto l253
}
{
switch buffer[position] {
case 'U':
if buffer[position] != rune('U') {
goto l248
goto l253
}
position++
if !_rules[rulehexQuad]() {
goto l248
goto l253
}
if !_rules[rulehexQuad]() {
goto l248
goto l253
}
break
case 'u':
if buffer[position] != rune('u') {
goto l248
goto l253
}
position++
if !_rules[rulehexQuad]() {
goto l248
goto l253
}
break
case '\\':
if buffer[position] != rune('\\') {
goto l248
goto l253
}
position++
break
case '/':
if buffer[position] != rune('/') {
goto l248
goto l253
}
position++
break
case '"':
if buffer[position] != rune('"') {
goto l248
goto l253
}
position++
break
case 'r':
if buffer[position] != rune('r') {
goto l248
goto l253
}
position++
break
case 'f':
if buffer[position] != rune('f') {
goto l248
goto l253
}
position++
break
case 'n':
if buffer[position] != rune('n') {
goto l248
goto l253
}
position++
break
case 't':
if buffer[position] != rune('t') {
goto l248
goto l253
}
position++
break
default:
if buffer[position] != rune('b') {
goto l248
goto l253
}
position++
break
}
}
add(ruleescaped, position254)
add(ruleescaped, position259)
}
}
l250:
add(rulebasicChar, position249)
l255:
add(rulebasicChar, position254)
}
return true
l248:
position, tokenIndex = position248, tokenIndex248
l253:
position, tokenIndex = position253, tokenIndex253
return false
},
/* 27 escaped <- <(escape ((&('U') ('U' hexQuad hexQuad)) | (&('u') ('u' hexQuad)) | (&('\\') '\\') | (&('/') '/') | (&('"') '"') | (&('r') 'r') | (&('f') 'f') | (&('n') 'n') | (&('t') 't') | (&('b') 'b')))> */
/* 28 escaped <- <(escape ((&('U') ('U' hexQuad hexQuad)) | (&('u') ('u' hexQuad)) | (&('\\') '\\') | (&('/') '/') | (&('"') '"') | (&('r') 'r') | (&('f') 'f') | (&('n') 'n') | (&('t') 't') | (&('b') 'b')))> */
nil,
/* 28 basicUnescaped <- <((&(' ' | '!') [ -!]) | (&('#' | '$' | '%' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '[') [#-[]) | (&(']' | '^' | '_' | '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | '{' | '|' | '}' | '~' | '\u007f' | '\u0080' | '\u0081' | '\u0082' | '\u0083' | '\u0084' | '\u0085' | '\u0086' | '\u0087' | '\u0088' | '\u0089' | '\u008a' | '\u008b' | '\u008c' | '\u008d' | '\u008e' | '\u008f' | '\u0090' | '\u0091' | '\u0092' | '\u0093' | '\u0094' | '\u0095' | '\u0096' | '\u0097' | '\u0098' | '\u0099' | '\u009a' | '\u009b' | '\u009c' | '\u009d' | '\u009e' | '\u009f' | '\u00a0' | '¡' | '¢' | '£' | '¤' | '¥' | '¦' | '§' | '¨' | '©' | 'ª' | '«' | '¬' | '\u00ad' | '®' | '¯' | '°' | '±' | '²' | '³' | '´' | 'µ' | '¶' | '·' | '¸' | '¹' | 'º' | '»' | '¼' | '½' | '¾' | '¿' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'Æ' | 'Ç' | 'È' | 'É' | 'Ê' | 'Ë' | 'Ì' | 'Í' | 'Î' | 'Ï' | 'Ð' | 'Ñ' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | '×' | 'Ø' | 'Ù' | 'Ú' | 'Û' | 'Ü' | 'Ý' | 'Þ' | 'ß' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'æ' | 'ç' | 'è' | 'é' | 'ê' | 'ë' | 'ì' | 'í' | 'î' | 'ï' | 'ð' | 'ñ' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | '÷' | 'ø' | 'ù' | 'ú' | 'û' | 'ü' | 'ý' | 'þ' | 'ÿ') []-\U0010ffff]))> */
/* 29 basicUnescaped <- <((&(' ' | '!') [ -!]) | (&('#' | '$' | '%' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '[') [#-[]) | (&(']' | '^' | '_' | '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | '{' | '|' | '}' | '~' | '\u007f' | '\u0080' | '\u0081' | '\u0082' | '\u0083' | '\u0084' | '\u0085' | '\u0086' | '\u0087' | '\u0088' | '\u0089' | '\u008a' | '\u008b' | '\u008c' | '\u008d' | '\u008e' | '\u008f' | '\u0090' | '\u0091' | '\u0092' | '\u0093' | '\u0094' | '\u0095' | '\u0096' | '\u0097' | '\u0098' | '\u0099' | '\u009a' | '\u009b' | '\u009c' | '\u009d' | '\u009e' | '\u009f' | '\u00a0' | '¡' | '¢' | '£' | '¤' | '¥' | '¦' | '§' | '¨' | '©' | 'ª' | '«' | '¬' | '\u00ad' | '®' | '¯' | '°' | '±' | '²' | '³' | '´' | 'µ' | '¶' | '·' | '¸' | '¹' | 'º' | '»' | '¼' | '½' | '¾' | '¿' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'Æ' | 'Ç' | 'È' | 'É' | 'Ê' | 'Ë' | 'Ì' | 'Í' | 'Î' | 'Ï' | 'Ð' | 'Ñ' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | '×' | 'Ø' | 'Ù' | 'Ú' | 'Û' | 'Ü' | 'Ý' | 'Þ' | 'ß' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'æ' | 'ç' | 'è' | 'é' | 'ê' | 'ë' | 'ì' | 'í' | 'î' | 'ï' | 'ð' | 'ñ' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | '÷' | 'ø' | 'ù' | 'ú' | 'û' | 'ü' | 'ý' | 'þ' | 'ÿ') []-\U0010ffff]))> */
nil,
/* 29 escape <- <'\\'> */
/* 30 escape <- <'\\'> */
func() bool {
position258, tokenIndex258 := position, tokenIndex
position263, tokenIndex263 := position, tokenIndex
{
position259 := position
position264 := position
if buffer[position] != rune('\\') {
goto l258
goto l263
}
position++
add(ruleescape, position259)
add(ruleescape, position264)
}
return true
l258:
position, tokenIndex = position258, tokenIndex258
l263:
position, tokenIndex = position263, tokenIndex263
return false
},
/* 30 mlBasicString <- <('"' '"' '"' mlBasicBody ('"' '"' '"') Action18)> */
/* 31 mlBasicString <- <('"' '"' '"' mlBasicBody ('"' '"' '"') Action19)> */
nil,
/* 31 mlBasicBody <- <((<(basicChar / newline)> Action19) / (escape newline wsnl))*> */
/* 32 mlBasicBody <- <((<(basicChar / newline)> Action20) / (escape newline wsnl))*> */
nil,
/* 32 literalString <- <('\'' <literalChar*> '\'' Action20)> */
/* 33 literalString <- <('\'' <literalChar*> '\'' Action21)> */
nil,
/* 33 literalChar <- <((&('\t') '\t') | (&(' ' | '!' | '"' | '#' | '$' | '%' | '&') [ -&]) | (&('(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '[' | '\\' | ']' | '^' | '_' | '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | '{' | '|' | '}' | '~' | '\u007f' | '\u0080' | '\u0081' | '\u0082' | '\u0083' | '\u0084' | '\u0085' | '\u0086' | '\u0087' | '\u0088' | '\u0089' | '\u008a' | '\u008b' | '\u008c' | '\u008d' | '\u008e' | '\u008f' | '\u0090' | '\u0091' | '\u0092' | '\u0093' | '\u0094' | '\u0095' | '\u0096' | '\u0097' | '\u0098' | '\u0099' | '\u009a' | '\u009b' | '\u009c' | '\u009d' | '\u009e' | '\u009f' | '\u00a0' | '¡' | '¢' | '£' | '¤' | '¥' | '¦' | '§' | '¨' | '©' | 'ª' | '«' | '¬' | '\u00ad' | '®' | '¯' | '°' | '±' | '²' | '³' | '´' | 'µ' | '¶' | '·' | '¸' | '¹' | 'º' | '»' | '¼' | '½' | '¾' | '¿' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'Æ' | 'Ç' | 'È' | 'É' | 'Ê' | 'Ë' | 'Ì' | 'Í' | 'Î' | 'Ï' | 'Ð' | 'Ñ' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | '×' | 'Ø' | 'Ù' | 'Ú' | 'Û' | 'Ü' | 'Ý' | 'Þ' | 'ß' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'æ' | 'ç' | 'è' | 'é' | 'ê' | 'ë' | 'ì' | 'í' | 'î' | 'ï' | 'ð' | 'ñ' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | '÷' | 'ø' | 'ù' | 'ú' | 'û' | 'ü' | 'ý' | 'þ' | 'ÿ') [(-\U0010ffff]))> */
/* 34 literalChar <- <((&('\t') '\t') | (&(' ' | '!' | '"' | '#' | '$' | '%' | '&') [ -&]) | (&('(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '[' | '\\' | ']' | '^' | '_' | '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | '{' | '|' | '}' | '~' | '\u007f' | '\u0080' | '\u0081' | '\u0082' | '\u0083' | '\u0084' | '\u0085' | '\u0086' | '\u0087' | '\u0088' | '\u0089' | '\u008a' | '\u008b' | '\u008c' | '\u008d' | '\u008e' | '\u008f' | '\u0090' | '\u0091' | '\u0092' | '\u0093' | '\u0094' | '\u0095' | '\u0096' | '\u0097' | '\u0098' | '\u0099' | '\u009a' | '\u009b' | '\u009c' | '\u009d' | '\u009e' | '\u009f' | '\u00a0' | '¡' | '¢' | '£' | '¤' | '¥' | '¦' | '§' | '¨' | '©' | 'ª' | '«' | '¬' | '\u00ad' | '®' | '¯' | '°' | '±' | '²' | '³' | '´' | 'µ' | '¶' | '·' | '¸' | '¹' | 'º' | '»' | '¼' | '½' | '¾' | '¿' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'Æ' | 'Ç' | 'È' | 'É' | 'Ê' | 'Ë' | 'Ì' | 'Í' | 'Î' | 'Ï' | 'Ð' | 'Ñ' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | '×' | 'Ø' | 'Ù' | 'Ú' | 'Û' | 'Ü' | 'Ý' | 'Þ' | 'ß' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'æ' | 'ç' | 'è' | 'é' | 'ê' | 'ë' | 'ì' | 'í' | 'î' | 'ï' | 'ð' | 'ñ' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | '÷' | 'ø' | 'ù' | 'ú' | 'û' | 'ü' | 'ý' | 'þ' | 'ÿ') [(-\U0010ffff]))> */
nil,
/* 34 mlLiteralString <- <('\'' '\'' '\'' <mlLiteralBody> ('\'' '\'' '\'') Action21)> */
/* 35 mlLiteralString <- <('\'' '\'' '\'' <mlLiteralBody> ('\'' '\'' '\'') Action22)> */
nil,
/* 35 mlLiteralBody <- <(!('\'' '\'' '\'') (mlLiteralChar / newline))*> */
/* 36 mlLiteralBody <- <(!('\'' '\'' '\'') (mlLiteralChar / newline))*> */
nil,
/* 36 mlLiteralChar <- <('\t' / [ -\U0010ffff])> */
/* 37 mlLiteralChar <- <('\t' / [ -\U0010ffff])> */
nil,
/* 37 hexdigit <- <((&('a' | 'b' | 'c' | 'd' | 'e' | 'f') [a-f]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F') [A-F]) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]))> */
/* 38 hexdigit <- <((&('a' | 'b' | 'c' | 'd' | 'e' | 'f') [a-f]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F') [A-F]) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]))> */
func() bool {
position267, tokenIndex267 := position, tokenIndex
position272, tokenIndex272 := position, tokenIndex
{
position268 := position
position273 := position
{
switch buffer[position] {
case 'a', 'b', 'c', 'd', 'e', 'f':
if c := buffer[position]; c < rune('a') || c > rune('f') {
goto l267
goto l272
}
position++
break
case 'A', 'B', 'C', 'D', 'E', 'F':
if c := buffer[position]; c < rune('A') || c > rune('F') {
goto l267
goto l272
}
position++
break
default:
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l267
goto l272
}
position++
break
}
}
add(rulehexdigit, position268)
add(rulehexdigit, position273)
}
return true
l267:
position, tokenIndex = position267, tokenIndex267
l272:
position, tokenIndex = position272, tokenIndex272
return false
},
/* 38 hexQuad <- <(hexdigit hexdigit hexdigit hexdigit)> */
/* 39 hexQuad <- <(hexdigit hexdigit hexdigit hexdigit)> */
func() bool {
position270, tokenIndex270 := position, tokenIndex
position275, tokenIndex275 := position, tokenIndex
{
position271 := position
position276 := position
if !_rules[rulehexdigit]() {
goto l270
goto l275
}
if !_rules[rulehexdigit]() {
goto l270
goto l275
}
if !_rules[rulehexdigit]() {
goto l270
goto l275
}
if !_rules[rulehexdigit]() {
goto l270
goto l275
}
add(rulehexQuad, position271)
add(rulehexQuad, position276)
}
return true
l270:
position, tokenIndex = position270, tokenIndex270
l275:
position, tokenIndex = position275, tokenIndex275
return false
},
/* 39 boolean <- <(('t' 'r' 'u' 'e') / ('f' 'a' 'l' 's' 'e'))> */
/* 40 boolean <- <(('t' 'r' 'u' 'e') / ('f' 'a' 'l' 's' 'e'))> */
nil,
/* 40 dateFullYear <- <digitQuad> */
/* 41 dateFullYear <- <digitQuad> */
nil,
/* 41 dateMonth <- <digitDual> */
/* 42 dateMonth <- <digitDual> */
nil,
/* 42 dateMDay <- <digitDual> */
/* 43 dateMDay <- <digitDual> */
nil,
/* 43 timeHour <- <digitDual> */
/* 44 timeHour <- <digitDual> */
func() bool {
position276, tokenIndex276 := position, tokenIndex
position281, tokenIndex281 := position, tokenIndex
{
position277 := position
position282 := position
if !_rules[ruledigitDual]() {
goto l276
goto l281
}
add(ruletimeHour, position277)
add(ruletimeHour, position282)
}
return true
l276:
position, tokenIndex = position276, tokenIndex276
l281:
position, tokenIndex = position281, tokenIndex281
return false
},
/* 44 timeMinute <- <digitDual> */
/* 45 timeMinute <- <digitDual> */
func() bool {
position278, tokenIndex278 := position, tokenIndex
position283, tokenIndex283 := position, tokenIndex
{
position279 := position
position284 := position
if !_rules[ruledigitDual]() {
goto l278
goto l283
}
add(ruletimeMinute, position279)
add(ruletimeMinute, position284)
}
return true
l278:
position, tokenIndex = position278, tokenIndex278
l283:
position, tokenIndex = position283, tokenIndex283
return false
},
/* 45 timeSecond <- <digitDual> */
/* 46 timeSecond <- <digitDual> */
nil,
/* 46 timeSecfrac <- <('.' digit+)> */
/* 47 timeSecfrac <- <('.' digit+)> */
nil,
/* 47 timeNumoffset <- <(('-' / '+') timeHour ':' timeMinute)> */
/* 48 timeNumoffset <- <(('-' / '+') timeHour ':' timeMinute)> */
nil,
/* 48 timeOffset <- <('Z' / timeNumoffset)> */
/* 49 timeOffset <- <('Z' / timeNumoffset)> */
nil,
/* 49 partialTime <- <(timeHour ':' timeMinute ':' timeSecond timeSecfrac?)> */
/* 50 partialTime <- <(timeHour ':' timeMinute ':' timeSecond timeSecfrac?)> */
func() bool {
position284, tokenIndex284 := position, tokenIndex
position289, tokenIndex289 := position, tokenIndex
{
position285 := position
position290 := position
if !_rules[ruletimeHour]() {
goto l284
goto l289
}
if buffer[position] != rune(':') {
goto l284
goto l289
}
position++
if !_rules[ruletimeMinute]() {
goto l284
goto l289
}
if buffer[position] != rune(':') {
goto l284
goto l289
}
position++
{
position286 := position
position291 := position
if !_rules[ruledigitDual]() {
goto l284
goto l289
}
add(ruletimeSecond, position286)
add(ruletimeSecond, position291)
}
{
position287, tokenIndex287 := position, tokenIndex
position292, tokenIndex292 := position, tokenIndex
{
position289 := position
position294 := position
if buffer[position] != rune('.') {
goto l287
goto l292
}
position++
if !_rules[ruledigit]() {
goto l287
goto l292
}
l290:
l295:
{
position291, tokenIndex291 := position, tokenIndex
position296, tokenIndex296 := position, tokenIndex
if !_rules[ruledigit]() {
goto l291
goto l296
}
goto l290
l291:
position, tokenIndex = position291, tokenIndex291
goto l295
l296:
position, tokenIndex = position296, tokenIndex296
}
add(ruletimeSecfrac, position289)
add(ruletimeSecfrac, position294)
}
goto l288
l287:
position, tokenIndex = position287, tokenIndex287
goto l293
l292:
position, tokenIndex = position292, tokenIndex292
}
l288:
add(rulepartialTime, position285)
l293:
add(rulepartialTime, position290)
}
return true
l284:
position, tokenIndex = position284, tokenIndex284
l289:
position, tokenIndex = position289, tokenIndex289
return false
},
/* 50 fullDate <- <(dateFullYear '-' dateMonth '-' dateMDay)> */
/* 51 fullDate <- <(dateFullYear '-' dateMonth '-' dateMDay)> */
nil,
/* 51 fullTime <- <(partialTime timeOffset)> */
/* 52 fullTime <- <(partialTime timeOffset?)> */
nil,
/* 52 datetime <- <((fullDate ('T' fullTime)?) / partialTime)> */
/* 53 datetime <- <((fullDate ('T' fullTime)?) / partialTime)> */
nil,
/* 53 digit <- <[0-9]> */
/* 54 digit <- <[0-9]> */
func() bool {
position295, tokenIndex295 := position, tokenIndex
position300, tokenIndex300 := position, tokenIndex
{
position296 := position
position301 := position
if c := buffer[position]; c < rune('0') || c > rune('9') {
goto l295
goto l300
}
position++
add(ruledigit, position296)
add(ruledigit, position301)
}
return true
l295:
position, tokenIndex = position295, tokenIndex295
l300:
position, tokenIndex = position300, tokenIndex300
return false
},
/* 54 digitDual <- <(digit digit)> */
/* 55 digitDual <- <(digit digit)> */
func() bool {
position297, tokenIndex297 := position, tokenIndex
position302, tokenIndex302 := position, tokenIndex
{
position298 := position
position303 := position
if !_rules[ruledigit]() {
goto l297
goto l302
}
if !_rules[ruledigit]() {
goto l297
goto l302
}
add(ruledigitDual, position298)
add(ruledigitDual, position303)
}
return true
l297:
position, tokenIndex = position297, tokenIndex297
l302:
position, tokenIndex = position302, tokenIndex302
return false
},
/* 55 digitQuad <- <(digitDual digitDual)> */
/* 56 digitQuad <- <(digitDual digitDual)> */
nil,
/* 56 array <- <('[' Action22 wsnl arrayValues? wsnl ']')> */
/* 57 array <- <('[' Action23 wsnl arrayValues? wsnl ']')> */
nil,
/* 57 arrayValues <- <(val Action23 (wsnl comment? wsnl arraySep wsnl comment? wsnl val Action24)* wsnl arraySep? wsnl comment?)> */
/* 58 arrayValues <- <(val Action24 (wsnl comment? wsnl arraySep wsnl comment? wsnl val Action25)* wsnl arraySep? wsnl comment?)> */
nil,
/* 58 arraySep <- <','> */
/* 59 arraySep <- <','> */
func() bool {
position302, tokenIndex302 := position, tokenIndex
position307, tokenIndex307 := position, tokenIndex
{
position303 := position
position308 := position
if buffer[position] != rune(',') {
goto l302
goto l307
}
position++
add(rulearraySep, position303)
add(rulearraySep, position308)
}
return true
l302:
position, tokenIndex = position302, tokenIndex302
l307:
position, tokenIndex = position307, tokenIndex307
return false
},
/* 60 Action0 <- <{ _ = buffer }> */
/* 61 Action0 <- <{ _ = buffer }> */
nil,
nil,
/* 63 Action1 <- <{ p.SetTableString(begin, end) }> */
nil,
/* 62 Action1 <- <{ p.SetTableString(begin, end) }> */
/* 64 Action2 <- <{ p.AddLineCount(end - begin) }> */
nil,
/* 63 Action2 <- <{ p.AddLineCount(end - begin) }> */
/* 65 Action3 <- <{ p.AddLineCount(end - begin) }> */
nil,
/* 64 Action3 <- <{ p.AddLineCount(end - begin) }> */
/* 66 Action4 <- <{ p.AddKeyValue() }> */
nil,
/* 65 Action4 <- <{ p.AddKeyValue() }> */
/* 67 Action5 <- <{ p.SetKey(p.buffer, begin, end) }> */
nil,
/* 66 Action5 <- <{ p.SetKey(p.buffer, begin, end) }> */
/* 68 Action6 <- <{ p.SetKey(p.buffer, begin, end) }> */
nil,
/* 67 Action6 <- <{ p.SetKey(p.buffer, begin-1, end+1) }> */
/* 69 Action7 <- <{ p.SetTime(begin, end) }> */
nil,
/* 68 Action7 <- <{ p.SetTime(begin, end) }> */
/* 70 Action8 <- <{ p.SetFloat64(begin, end) }> */
nil,
/* 69 Action8 <- <{ p.SetFloat64(begin, end) }> */
/* 71 Action9 <- <{ p.SetInt64(begin, end) }> */
nil,
/* 70 Action9 <- <{ p.SetInt64(begin, end) }> */
/* 72 Action10 <- <{ p.SetString(begin, end) }> */
nil,
/* 71 Action10 <- <{ p.SetString(begin, end) }> */
/* 73 Action11 <- <{ p.SetBool(begin, end) }> */
nil,
/* 72 Action11 <- <{ p.SetBool(begin, end) }> */
/* 74 Action12 <- <{ p.SetArray(begin, end) }> */
nil,
/* 73 Action12 <- <{ p.SetArray(begin, end) }> */
/* 75 Action13 <- <{ p.SetTable(p.buffer, begin, end) }> */
nil,
/* 74 Action13 <- <{ p.SetTable(p.buffer, begin, end) }> */
/* 76 Action14 <- <{ p.SetArrayTable(p.buffer, begin, end) }> */
nil,
/* 75 Action14 <- <{ p.SetArrayTable(p.buffer, begin, end) }> */
/* 77 Action15 <- <{ p.StartInlineTable() }> */
nil,
/* 76 Action15 <- <{ p.StartInlineTable() }> */
/* 78 Action16 <- <{ p.EndInlineTable() }> */
nil,
/* 77 Action16 <- <{ p.EndInlineTable() }> */
/* 79 Action17 <- <{ p.AddTableKey() }> */
nil,
/* 78 Action17 <- <{ p.SetBasicString(p.buffer, begin, end) }> */
/* 80 Action18 <- <{ p.SetBasicString(p.buffer, begin, end) }> */
nil,
/* 79 Action18 <- <{ p.SetMultilineString() }> */
/* 81 Action19 <- <{ p.SetMultilineString() }> */
nil,
/* 80 Action19 <- <{ p.AddMultilineBasicBody(p.buffer, begin, end) }> */
/* 82 Action20 <- <{ p.AddMultilineBasicBody(p.buffer, begin, end) }> */
nil,
/* 81 Action20 <- <{ p.SetLiteralString(p.buffer, begin, end) }> */
/* 83 Action21 <- <{ p.SetLiteralString(p.buffer, begin, end) }> */
nil,
/* 82 Action21 <- <{ p.SetMultilineLiteralString(p.buffer, begin, end) }> */
/* 84 Action22 <- <{ p.SetMultilineLiteralString(p.buffer, begin, end) }> */
nil,
/* 83 Action22 <- <{ p.StartArray() }> */
/* 85 Action23 <- <{ p.StartArray() }> */
nil,
/* 84 Action23 <- <{ p.AddArrayVal() }> */
/* 86 Action24 <- <{ p.AddArrayVal() }> */
nil,
/* 85 Action24 <- <{ p.AddArrayVal() }> */
/* 87 Action25 <- <{ p.AddArrayVal() }> */
nil,
}
p.rules = _rules
......
vendor/golang.org/x/net/idna/idna.go 0 → 100644
View file @ 5bee5d69
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package idna implements IDNA2008 using the compatibility processing
// defined by UTS (Unicode Technical Standard) #46, which defines a standard to
// deal with the transition from IDNA2003.
//
// IDNA2008 (Internationalized Domain Names for Applications), is defined in RFC
// 5890, RFC 5891, RFC 5892, RFC 5893 and RFC 5894.
// UTS #46 is defined in http://www.unicode.org/reports/tr46.
// See http://unicode.org/cldr/utility/idna.jsp for a visualization of the
// differences between these two standards.
package idna // import "golang.org/x/net/idna"
import (
"fmt"
"strings"
"unicode/utf8"
"golang.org/x/text/secure/bidirule"
"golang.org/x/text/unicode/bidi"
"golang.org/x/text/unicode/norm"
)
// NOTE: Unlike common practice in Go APIs, the functions will return a
// sanitized domain name in case of errors. Browsers sometimes use a partially
// evaluated string as lookup.
// TODO: the current error handling is, in my opinion, the least opinionated.
// Other strategies are also viable, though:
// Option 1) Return an empty string in case of error, but allow the user to
// specify explicitly which errors to ignore.
// Option 2) Return the partially evaluated string if it is itself a valid
// string, otherwise return the empty string in case of error.
// Option 3) Option 1 and 2.
// Option 4) Always return an empty string for now and implement Option 1 as
// needed, and document that the return string may not be empty in case of
// error in the future.
// I think Option 1 is best, but it is quite opinionated.
// ToASCII is a wrapper for Punycode.ToASCII.
func ToASCII(s string) (string, error) {
return Punycode.process(s, true)
}
// ToUnicode is a wrapper for Punycode.ToUnicode.
func ToUnicode(s string) (string, error) {
return Punycode.process(s, false)
}
// An Option configures a Profile at creation time.
type Option func(*options)
// Transitional sets a Profile to use the Transitional mapping as defined in UTS
// #46. This will cause, for example, "ß" to be mapped to "ss". Using the
// transitional mapping provides a compromise between IDNA2003 and IDNA2008
// compatibility. It is used by most browsers when resolving domain names. This
// option is only meaningful if combined with MapForLookup.
func Transitional(transitional bool) Option {
return func(o *options) { o.transitional = true }
}
// VerifyDNSLength sets whether a Profile should fail if any of the IDN parts
// are longer than allowed by the RFC.
func VerifyDNSLength(verify bool) Option {
return func(o *options) { o.verifyDNSLength = verify }
}
// RemoveLeadingDots removes leading label separators. Leading runes that map to
// dots, such as U+3002 IDEOGRAPHIC FULL STOP, are removed as well.
//
// This is the behavior suggested by the UTS #46 and is adopted by some
// browsers.
func RemoveLeadingDots(remove bool) Option {
return func(o *options) { o.removeLeadingDots = remove }
}
// ValidateLabels sets whether to check the mandatory label validation criteria
// as defined in Section 5.4 of RFC 5891. This includes testing for correct use
// of hyphens ('-'), normalization, validity of runes, and the context rules.
func ValidateLabels(enable bool) Option {
return func(o *options) {
// Don't override existing mappings, but set one that at least checks
// normalization if it is not set.
if o.mapping == nil && enable {
o.mapping = normalize
}
o.trie = trie
o.validateLabels = enable
o.fromPuny = validateFromPunycode
}
}
// StrictDomainName limits the set of permissible ASCII characters to those
// allowed in domain names as defined in RFC 1034 (A-Z, a-z, 0-9 and the
// hyphen). This is set by default for MapForLookup and ValidateForRegistration.
//
// This option is useful, for instance, for browsers that allow characters
// outside this range, for example a '_' (U+005F LOW LINE). See
// http://www.rfc-editor.org/std/std3.txt for more details This option
// corresponds to the UseSTD3ASCIIRules option in UTS #46.
func StrictDomainName(use bool) Option {
return func(o *options) {
o.trie = trie
o.useSTD3Rules = use
o.fromPuny = validateFromPunycode
}
}
// NOTE: the following options pull in tables. The tables should not be linked
// in as long as the options are not used.
// BidiRule enables the Bidi rule as defined in RFC 5893. Any application
// that relies on proper validation of labels should include this rule.
func BidiRule() Option {
return func(o *options) { o.bidirule = bidirule.ValidString }
}
// ValidateForRegistration sets validation options to verify that a given IDN is
// properly formatted for registration as defined by Section 4 of RFC 5891.
func ValidateForRegistration() Option {
return func(o *options) {
o.mapping = validateRegistration
StrictDomainName(true)(o)
ValidateLabels(true)(o)
VerifyDNSLength(true)(o)
BidiRule()(o)
}
}
// MapForLookup sets validation and mapping options such that a given IDN is
// transformed for domain name lookup according to the requirements set out in
// Section 5 of RFC 5891. The mappings follow the recommendations of RFC 5894,
// RFC 5895 and UTS 46. It does not add the Bidi Rule. Use the BidiRule option
// to add this check.
//
// The mappings include normalization and mapping case, width and other
// compatibility mappings.
func MapForLookup() Option {
return func(o *options) {
o.mapping = validateAndMap
StrictDomainName(true)(o)
ValidateLabels(true)(o)
}
}
type options struct {
transitional bool
useSTD3Rules bool
validateLabels bool
verifyDNSLength bool
removeLeadingDots bool
trie *idnaTrie
// fromPuny calls validation rules when converting A-labels to U-labels.
fromPuny func(p *Profile, s string) error
// mapping implements a validation and mapping step as defined in RFC 5895
// or UTS 46, tailored to, for example, domain registration or lookup.
mapping func(p *Profile, s string) (mapped string, isBidi bool, err error)
// bidirule, if specified, checks whether s conforms to the Bidi Rule
// defined in RFC 5893.
bidirule func(s string) bool
}
// A Profile defines the configuration of an IDNA mapper.
type Profile struct {
options
}
func apply(o *options, opts []Option) {
for _, f := range opts {
f(o)
}
}
// New creates a new Profile.
//
// With no options, the returned Profile is the most permissive and equals the
// Punycode Profile. Options can be passed to further restrict the Profile. The
// MapForLookup and ValidateForRegistration options set a collection of options,
// for lookup and registration purposes respectively, which can be tailored by
// adding more fine-grained options, where later options override earlier
// options.
func New(o ...Option) *Profile {
p := &Profile{}
apply(&p.options, o)
return p
}
// ToASCII converts a domain or domain label to its ASCII form. For example,
// ToASCII("bücher.example.com") is "xn--bcher-kva.example.com", and
// ToASCII("golang") is "golang". If an error is encountered it will return
// an error and a (partially) processed result.
func (p *Profile) ToASCII(s string) (string, error) {
return p.process(s, true)
}
// ToUnicode converts a domain or domain label to its Unicode form. For example,
// ToUnicode("xn--bcher-kva.example.com") is "bücher.example.com", and
// ToUnicode("golang") is "golang". If an error is encountered it will return
// an error and a (partially) processed result.
func (p *Profile) ToUnicode(s string) (string, error) {
pp := *p
pp.transitional = false
return pp.process(s, false)
}
// String reports a string with a description of the profile for debugging
// purposes. The string format may change with different versions.
func (p *Profile) String() string {
s := ""
if p.transitional {
s = "Transitional"
} else {
s = "NonTransitional"
}
if p.useSTD3Rules {
s += ":UseSTD3Rules"
}
if p.validateLabels {
s += ":ValidateLabels"
}
if p.verifyDNSLength {
s += ":VerifyDNSLength"
}
return s
}
var (
// Punycode is a Profile that does raw punycode processing with a minimum
// of validation.
Punycode *Profile = punycode
// Lookup is the recommended profile for looking up domain names, according
// to Section 5 of RFC 5891. The exact configuration of this profile may
// change over time.
Lookup *Profile = lookup
// Display is the recommended profile for displaying domain names.
// The configuration of this profile may change over time.
Display *Profile = display
// Registration is the recommended profile for checking whether a given
// IDN is valid for registration, according to Section 4 of RFC 5891.
Registration *Profile = registration
punycode = &Profile{}
lookup = &Profile{options{
transitional: true,
useSTD3Rules: true,
validateLabels: true,
trie: trie,
fromPuny: validateFromPunycode,
mapping: validateAndMap,
bidirule: bidirule.ValidString,
}}
display = &Profile{options{
useSTD3Rules: true,
validateLabels: true,
trie: trie,
fromPuny: validateFromPunycode,
mapping: validateAndMap,
bidirule: bidirule.ValidString,
}}
registration = &Profile{options{
useSTD3Rules: true,
validateLabels: true,
verifyDNSLength: true,
trie: trie,
fromPuny: validateFromPunycode,
mapping: validateRegistration,
bidirule: bidirule.ValidString,
}}
// TODO: profiles
// Register: recommended for approving domain names: don't do any mappings
// but rather reject on invalid input. Bundle or block deviation characters.
)
type labelError struct{ label, code_ string }
func (e labelError) code() string { return e.code_ }
func (e labelError) Error() string {
return fmt.Sprintf("idna: invalid label %q", e.label)
}
type runeError rune
func (e runeError) code() string { return "P1" }
func (e runeError) Error() string {
return fmt.Sprintf("idna: disallowed rune %U", e)
}
// process implements the algorithm described in section 4 of UTS #46,
// see http://www.unicode.org/reports/tr46.
func (p *Profile) process(s string, toASCII bool) (string, error) {
var err error
var isBidi bool
if p.mapping != nil {
s, isBidi, err = p.mapping(p, s)
}
// Remove leading empty labels.
if p.removeLeadingDots {
for ; len(s) > 0 && s[0] == '.'; s = s[1:] {
}
}
// TODO: allow for a quick check of the tables data.
// It seems like we should only create this error on ToASCII, but the
// UTS 46 conformance tests suggests we should always check this.
if err == nil && p.verifyDNSLength && s == "" {
err = &labelError{s, "A4"}
}
labels := labelIter{orig: s}
for ; !labels.done(); labels.next() {
label := labels.label()
if label == "" {
// Empty labels are not okay. The label iterator skips the last
// label if it is empty.
if err == nil && p.verifyDNSLength {
err = &labelError{s, "A4"}
}
continue
}
if strings.HasPrefix(label, acePrefix) {
u, err2 := decode(label[len(acePrefix):])
if err2 != nil {
if err == nil {
err = err2
}
// Spec says keep the old label.
continue
}
isBidi = isBidi || bidirule.DirectionString(u) != bidi.LeftToRight
labels.set(u)
if err == nil && p.validateLabels {
err = p.fromPuny(p, u)
}
if err == nil {
// This should be called on NonTransitional, according to the
// spec, but that currently does not have any effect. Use the
// original profile to preserve options.
err = p.validateLabel(u)
}
} else if err == nil {
err = p.validateLabel(label)
}
}
if isBidi && p.bidirule != nil && err == nil {
for labels.reset(); !labels.done(); labels.next() {
if !p.bidirule(labels.label()) {
err = &labelError{s, "B"}
break
}
}
}
if toASCII {
for labels.reset(); !labels.done(); labels.next() {
label := labels.label()
if !ascii(label) {
a, err2 := encode(acePrefix, label)
if err == nil {
err = err2
}
label = a
labels.set(a)
}
n := len(label)
if p.verifyDNSLength && err == nil && (n == 0 || n > 63) {
err = &labelError{label, "A4"}
}
}
}
s = labels.result()
if toASCII && p.verifyDNSLength && err == nil {
// Compute the length of the domain name minus the root label and its dot.
n := len(s)
if n > 0 && s[n-1] == '.' {
n--
}
if len(s) < 1 || n > 253 {
err = &labelError{s, "A4"}
}
}
return s, err
}
func normalize(p *Profile, s string) (mapped string, isBidi bool, err error) {
// TODO: consider first doing a quick check to see if any of these checks
// need to be done. This will make it slower in the general case, but
// faster in the common case.
mapped = norm.NFC.String(s)
isBidi = bidirule.DirectionString(mapped) == bidi.RightToLeft
return mapped, isBidi, nil
}
func validateRegistration(p *Profile, s string) (idem string, bidi bool, err error) {
// TODO: filter need for normalization in loop below.
if !norm.NFC.IsNormalString(s) {
return s, false, &labelError{s, "V1"}
}
for i := 0; i < len(s); {
v, sz := trie.lookupString(s[i:])
if sz == 0 {
return s, bidi, runeError(utf8.RuneError)
}
bidi = bidi || info(v).isBidi(s[i:])
// Copy bytes not copied so far.
switch p.simplify(info(v).category()) {
// TODO: handle the NV8 defined in the Unicode idna data set to allow
// for strict conformance to IDNA2008.
case valid, deviation:
case disallowed, mapped, unknown, ignored:
r, _ := utf8.DecodeRuneInString(s[i:])
return s, bidi, runeError(r)
}
i += sz
}
return s, bidi, nil
}
func (c info) isBidi(s string) bool {
if !c.isMapped() {
return c&attributesMask == rtl
}
// TODO: also store bidi info for mapped data. This is possible, but a bit
// cumbersome and not for the common case.
p, _ := bidi.LookupString(s)
switch p.Class() {
case bidi.R, bidi.AL, bidi.AN:
return true
}
return false
}
func validateAndMap(p *Profile, s string) (vm string, bidi bool, err error) {
var (
b []byte
k int
)
// combinedInfoBits contains the or-ed bits of all runes. We use this
// to derive the mayNeedNorm bit later. This may trigger normalization
// overeagerly, but it will not do so in the common case. The end result
// is another 10% saving on BenchmarkProfile for the common case.
var combinedInfoBits info
for i := 0; i < len(s); {
v, sz := trie.lookupString(s[i:])
if sz == 0 {
b = append(b, s[k:i]...)
b = append(b, "\ufffd"...)
k = len(s)
if err == nil {
err = runeError(utf8.RuneError)
}
break
}
combinedInfoBits |= info(v)
bidi = bidi || info(v).isBidi(s[i:])
start := i
i += sz
// Copy bytes not copied so far.
switch p.simplify(info(v).category()) {
case valid:
continue
case disallowed:
if err == nil {
r, _ := utf8.DecodeRuneInString(s[start:])
err = runeError(r)
}
continue
case mapped, deviation:
b = append(b, s[k:start]...)
b = info(v).appendMapping(b, s[start:i])
case ignored:
b = append(b, s[k:start]...)
// drop the rune
case unknown:
b = append(b, s[k:start]...)
b = append(b, "\ufffd"...)
}
k = i
}
if k == 0 {
// No changes so far.
if combinedInfoBits&mayNeedNorm != 0 {
s = norm.NFC.String(s)
}
} else {
b = append(b, s[k:]...)
if norm.NFC.QuickSpan(b) != len(b) {
b = norm.NFC.Bytes(b)
}
// TODO: the punycode converters require strings as input.
s = string(b)
}
return s, bidi, err
}
// A labelIter allows iterating over domain name labels.
type labelIter struct {
orig string
slice []string
curStart int
curEnd int
i int
}
func (l *labelIter) reset() {
l.curStart = 0
l.curEnd = 0
l.i = 0
}
func (l *labelIter) done() bool {
return l.curStart >= len(l.orig)
}
func (l *labelIter) result() string {
if l.slice != nil {
return strings.Join(l.slice, ".")
}
return l.orig
}
func (l *labelIter) label() string {
if l.slice != nil {
return l.slice[l.i]
}
p := strings.IndexByte(l.orig[l.curStart:], '.')
l.curEnd = l.curStart + p
if p == -1 {
l.curEnd = len(l.orig)
}
return l.orig[l.curStart:l.curEnd]
}
// next sets the value to the next label. It skips the last label if it is empty.
func (l *labelIter) next() {
l.i++
if l.slice != nil {
if l.i >= len(l.slice) || l.i == len(l.slice)-1 && l.slice[l.i] == "" {
l.curStart = len(l.orig)
}
} else {
l.curStart = l.curEnd + 1
if l.curStart == len(l.orig)-1 && l.orig[l.curStart] == '.' {
l.curStart = len(l.orig)
}
}
}
func (l *labelIter) set(s string) {
if l.slice == nil {
l.slice = strings.Split(l.orig, ".")
}
l.slice[l.i] = s
}
// acePrefix is the ASCII Compatible Encoding prefix.
const acePrefix = "xn--"
func (p *Profile) simplify(cat category) category {
switch cat {
case disallowedSTD3Mapped:
if p.useSTD3Rules {
cat = disallowed
} else {
cat = mapped
}
case disallowedSTD3Valid:
if p.useSTD3Rules {
cat = disallowed
} else {
cat = valid
}
case deviation:
if !p.transitional {
cat = valid
}
case validNV8, validXV8:
// TODO: handle V2008
cat = valid
}
return cat
}
func validateFromPunycode(p *Profile, s string) error {
if !norm.NFC.IsNormalString(s) {
return &labelError{s, "V1"}
}
// TODO: detect whether string may have to be normalized in the following
// loop.
for i := 0; i < len(s); {
v, sz := trie.lookupString(s[i:])
if sz == 0 {
return runeError(utf8.RuneError)
}
if c := p.simplify(info(v).category()); c != valid && c != deviation {
return &labelError{s, "V6"}
}
i += sz
}
return nil
}
const (
zwnj = "\u200c"
zwj = "\u200d"
)
type joinState int8
const (
stateStart joinState = iota
stateVirama
stateBefore
stateBeforeVirama
stateAfter
stateFAIL
)
var joinStates = [][numJoinTypes]joinState{
stateStart: {
joiningL: stateBefore,
joiningD: stateBefore,
joinZWNJ: stateFAIL,
joinZWJ: stateFAIL,
joinVirama: stateVirama,
},
stateVirama: {
joiningL: stateBefore,
joiningD: stateBefore,
},
stateBefore: {
joiningL: stateBefore,
joiningD: stateBefore,
joiningT: stateBefore,
joinZWNJ: stateAfter,
joinZWJ: stateFAIL,
joinVirama: stateBeforeVirama,
},
stateBeforeVirama: {
joiningL: stateBefore,
joiningD: stateBefore,
joiningT: stateBefore,
},
stateAfter: {
joiningL: stateFAIL,
joiningD: stateBefore,
joiningT: stateAfter,
joiningR: stateStart,
joinZWNJ: stateFAIL,
joinZWJ: stateFAIL,
joinVirama: stateAfter, // no-op as we can't accept joiners here
},
stateFAIL: {
0: stateFAIL,
joiningL: stateFAIL,
joiningD: stateFAIL,
joiningT: stateFAIL,
joiningR: stateFAIL,
joinZWNJ: stateFAIL,
joinZWJ: stateFAIL,
joinVirama: stateFAIL,
},
}
// validateLabel validates the criteria from Section 4.1. Item 1, 4, and 6 are
// already implicitly satisfied by the overall implementation.
func (p *Profile) validateLabel(s string) (err error) {
if s == "" {
if p.verifyDNSLength {
return &labelError{s, "A4"}
}
return nil
}
if !p.validateLabels {
return nil
}
trie := p.trie // p.validateLabels is only set if trie is set.
if len(s) > 4 && s[2] == '-' && s[3] == '-' {
return &labelError{s, "V2"}
}
if s[0] == '-' || s[len(s)-1] == '-' {
return &labelError{s, "V3"}
}
// TODO: merge the use of this in the trie.
v, sz := trie.lookupString(s)
x := info(v)
if x.isModifier() {
return &labelError{s, "V5"}
}
// Quickly return in the absence of zero-width (non) joiners.
if strings.Index(s, zwj) == -1 && strings.Index(s, zwnj) == -1 {
return nil
}
st := stateStart
for i := 0; ; {
jt := x.joinType()
if s[i:i+sz] == zwj {
jt = joinZWJ
} else if s[i:i+sz] == zwnj {
jt = joinZWNJ
}
st = joinStates[st][jt]
if x.isViramaModifier() {
st = joinStates[st][joinVirama]
}
if i += sz; i == len(s) {
break
}
v, sz = trie.lookupString(s[i:])
x = info(v)
}
if st == stateFAIL || st == stateAfter {
return &labelError{s, "C"}
}
return nil
}
func ascii(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] >= utf8.RuneSelf {
return false
}
}
return true
}
vendor/golang.org/x/net/idna/punycode.go 0 → 100644
View file @ 5bee5d69
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package idna
// This file implements the Punycode algorithm from RFC 3492.
import (
"math"
"strings"
"unicode/utf8"
)
// These parameter values are specified in section 5.
//
// All computation is done with int32s, so that overflow behavior is identical
// regardless of whether int is 32-bit or 64-bit.
const (
base int32 = 36
damp int32 = 700
initialBias int32 = 72
initialN int32 = 128
skew int32 = 38
tmax int32 = 26
tmin int32 = 1
)
func punyError(s string) error { return &labelError{s, "A3"} }
// decode decodes a string as specified in section 6.2.
func decode(encoded string) (string, error) {
if encoded == "" {
return "", nil
}
pos := 1 + strings.LastIndex(encoded, "-")
if pos == 1 {
return "", punyError(encoded)
}
if pos == len(encoded) {
return encoded[:len(encoded)-1], nil
}
output := make([]rune, 0, len(encoded))
if pos != 0 {
for _, r := range encoded[:pos-1] {
output = append(output, r)
}
}
i, n, bias := int32(0), initialN, initialBias
for pos < len(encoded) {
oldI, w := i, int32(1)
for k := base; ; k += base {
if pos == len(encoded) {
return "", punyError(encoded)
}
digit, ok := decodeDigit(encoded[pos])
if !ok {
return "", punyError(encoded)
}
pos++
i += digit * w
if i < 0 {
return "", punyError(encoded)
}
t := k - bias
if t < tmin {
t = tmin
} else if t > tmax {
t = tmax
}
if digit < t {
break
}
w *= base - t
if w >= math.MaxInt32/base {
return "", punyError(encoded)
}
}
x := int32(len(output) + 1)
bias = adapt(i-oldI, x, oldI == 0)
n += i / x
i %= x
if n > utf8.MaxRune || len(output) >= 1024 {
return "", punyError(encoded)
}
output = append(output, 0)
copy(output[i+1:], output[i:])
output[i] = n
i++
}
return string(output), nil
}
// encode encodes a string as specified in section 6.3 and prepends prefix to
// the result.
//
// The "while h < length(input)" line in the specification becomes "for
// remaining != 0" in the Go code, because len(s) in Go is in bytes, not runes.
func encode(prefix, s string) (string, error) {
output := make([]byte, len(prefix), len(prefix)+1+2*len(s))
copy(output, prefix)
delta, n, bias := int32(0), initialN, initialBias
b, remaining := int32(0), int32(0)
for _, r := range s {
if r < 0x80 {
b++
output = append(output, byte(r))
} else {
remaining++
}
}
h := b
if b > 0 {
output = append(output, '-')
}
for remaining != 0 {
m := int32(0x7fffffff)
for _, r := range s {
if m > r && r >= n {
m = r
}
}
delta += (m - n) * (h + 1)
if delta < 0 {
return "", punyError(s)
}
n = m
for _, r := range s {
if r < n {
delta++
if delta < 0 {
return "", punyError(s)
}
continue
}
if r > n {
continue
}
q := delta
for k := base; ; k += base {
t := k - bias
if t < tmin {
t = tmin
} else if t > tmax {
t = tmax
}
if q < t {
break
}
output = append(output, encodeDigit(t+(q-t)%(base-t)))
q = (q - t) / (base - t)
}
output = append(output, encodeDigit(q))
bias = adapt(delta, h+1, h == b)
delta = 0
h++
remaining--
}
delta++
n++
}
return string(output), nil
}
func decodeDigit(x byte) (digit int32, ok bool) {
switch {
case '0' <= x && x <= '9':
return int32(x - ('0' - 26)), true
case 'A' <= x && x <= 'Z':
return int32(x - 'A'), true
case 'a' <= x && x <= 'z':
return int32(x - 'a'), true
}
return 0, false
}
func encodeDigit(digit int32) byte {
switch {
case 0 <= digit && digit < 26:
return byte(digit + 'a')
case 26 <= digit && digit < 36:
return byte(digit + ('0' - 26))
}
panic("idna: internal error in punycode encoding")
}
// adapt is the bias adaptation function specified in section 6.1.
func adapt(delta, numPoints int32, firstTime bool) int32 {
if firstTime {
delta /= damp
} else {
delta /= 2
}
delta += delta / numPoints
k := int32(0)
for delta > ((base-tmin)*tmax)/2 {
delta /= base - tmin
k += base
}
return k + (base-tmin+1)*delta/(delta+skew)
}
vendor/golang.org/x/net/idna/tables.go 0 → 100644
View file @ 5bee5d69
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/golang.org/x/net/idna/trie.go 0 → 100644
View file @ 5bee5d69
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package idna
// appendMapping appends the mapping for the respective rune. isMapped must be
// true. A mapping is a categorization of a rune as defined in UTS #46.
func (c info) appendMapping(b []byte, s string) []byte {
index := int(c >> indexShift)
if c&xorBit == 0 {
s := mappings[index:]
return append(b, s[1:s[0]+1]...)
}
b = append(b, s...)
if c&inlineXOR == inlineXOR {
// TODO: support and handle two-byte inline masks
b[len(b)-1] ^= byte(index)
} else {
for p := len(b) - int(xorData[index]); p < len(b); p++ {
index++
b[p] ^= xorData[index]
}
}
return b
}
// Sparse block handling code.
type valueRange struct {
value uint16 // header: value:stride
lo, hi byte // header: lo:n
}
type sparseBlocks struct {
values []valueRange
offset []uint16
}
var idnaSparse = sparseBlocks{
values: idnaSparseValues[:],
offset: idnaSparseOffset[:],
}
// Don't use newIdnaTrie to avoid unconditional linking in of the table.
var trie = &idnaTrie{}
// lookup determines the type of block n and looks up the value for b.
// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
// is a list of ranges with an accompanying value. Given a matching range r,
// the value for b is by r.value + (b - r.lo) * stride.
func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
offset := t.offset[n]
header := t.values[offset]
lo := offset + 1
hi := lo + uint16(header.lo)
for lo < hi {
m := lo + (hi-lo)/2
r := t.values[m]
if r.lo <= b && b <= r.hi {
return r.value + uint16(b-r.lo)*header.value
}
if b < r.lo {
hi = m
} else {
lo = m + 1
}
}
return 0
}
vendor/golang.org/x/net/idna/trieval.go 0 → 100644
View file @ 5bee5d69
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
package idna
// This file contains definitions for interpreting the trie value of the idna
// trie generated by "go run gen*.go". It is shared by both the generator
// program and the resultant package. Sharing is achieved by the generator
// copying gen_trieval.go to trieval.go and changing what's above this comment.
// info holds information from the IDNA mapping table for a single rune. It is
// the value returned by a trie lookup. In most cases, all information fits in
// a 16-bit value. For mappings, this value may contain an index into a slice
// with the mapped string. Such mappings can consist of the actual mapped value
// or an XOR pattern to be applied to the bytes of the UTF8 encoding of the
// input rune. This technique is used by the cases packages and reduces the
// table size significantly.
//
// The per-rune values have the following format:
//
// if mapped {
// if inlinedXOR {
// 15..13 inline XOR marker
// 12..11 unused
// 10..3 inline XOR mask
// } else {
// 15..3 index into xor or mapping table
// }
// } else {
// 15..14 unused
// 13 mayNeedNorm
// 12..11 attributes
// 10..8 joining type
// 7..3 category type
// }
// 2 use xor pattern
// 1..0 mapped category
//
// See the definitions below for a more detailed description of the various
// bits.
type info uint16
const (
catSmallMask = 0x3
catBigMask = 0xF8
indexShift = 3
xorBit = 0x4 // interpret the index as an xor pattern
inlineXOR = 0xE000 // These bits are set if the XOR pattern is inlined.
joinShift = 8
joinMask = 0x07
// Attributes
attributesMask = 0x1800
viramaModifier = 0x1800
modifier = 0x1000
rtl = 0x0800
mayNeedNorm = 0x2000
)
// A category corresponds to a category defined in the IDNA mapping table.
type category uint16
const (
unknown category = 0 // not currently defined in unicode.
mapped category = 1
disallowedSTD3Mapped category = 2
deviation category = 3
)
const (
valid category = 0x08
validNV8 category = 0x18
validXV8 category = 0x28
disallowed category = 0x40
disallowedSTD3Valid category = 0x80
ignored category = 0xC0
)
// join types and additional rune information
const (
joiningL = (iota + 1)
joiningD
joiningT
joiningR
//the following types are derived during processing
joinZWJ
joinZWNJ
joinVirama
numJoinTypes
)
func (c info) isMapped() bool {
return c&0x3 != 0
}
func (c info) category() category {
small := c & catSmallMask
if small != 0 {
return category(small)
}
return category(c & catBigMask)
}
func (c info) joinType() info {
if c.isMapped() {
return 0
}
return (c >> joinShift) & joinMask
}
func (c info) isModifier() bool {
return c&(modifier|catSmallMask) == modifier
}
func (c info) isViramaModifier() bool {
return c&(attributesMask|catSmallMask) == viramaModifier
}
vendor/golang.org/x/text/secure/bidirule/bidirule.go 0 → 100644
View file @ 5bee5d69
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package bidirule implements the Bidi Rule defined by RFC 5893.
//
// This package is under development. The API may change without notice and
// without preserving backward compatibility.
package bidirule
import (
"errors"
"unicode/utf8"
"golang.org/x/text/transform"
"golang.org/x/text/unicode/bidi"
)
// This file contains an implementation of RFC 5893: Right-to-Left Scripts for
// Internationalized Domain Names for Applications (IDNA)
//
// A label is an individual component of a domain name. Labels are usually
// shown separated by dots; for example, the domain name "www.example.com" is
// composed of three labels: "www", "example", and "com".
//
// An RTL label is a label that contains at least one character of class R, AL,
// or AN. An LTR label is any label that is not an RTL label.
//
// A "Bidi domain name" is a domain name that contains at least one RTL label.
//
// The following guarantees can be made based on the above:
//
// o In a domain name consisting of only labels that satisfy the rule,
// the requirements of Section 3 are satisfied. Note that even LTR
// labels and pure ASCII labels have to be tested.
//
// o In a domain name consisting of only LDH labels (as defined in the
// Definitions document [RFC5890]) and labels that satisfy the rule,
// the requirements of Section 3 are satisfied as long as a label
// that starts with an ASCII digit does not come after a
// right-to-left label.
//
// No guarantee is given for other combinations.
// ErrInvalid indicates a label is invalid according to the Bidi Rule.
var ErrInvalid = errors.New("bidirule: failed Bidi Rule")
type ruleState uint8
const (
ruleInitial ruleState = iota
ruleLTR
ruleLTRFinal
ruleRTL
ruleRTLFinal
ruleInvalid
)
type ruleTransition struct {
next ruleState
mask uint16
}
var transitions = [...][2]ruleTransition{
// [2.1] The first character must be a character with Bidi property L, R, or
// AL. If it has the R or AL property, it is an RTL label; if it has the L
// property, it is an LTR label.
ruleInitial: {
{ruleLTRFinal, 1 << bidi.L},
{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL},
},
ruleRTL: {
// [2.3] In an RTL label, the end of the label must be a character with
// Bidi property R, AL, EN, or AN, followed by zero or more characters
// with Bidi property NSM.
{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL | 1<<bidi.EN | 1<<bidi.AN},
// [2.2] In an RTL label, only characters with the Bidi properties R,
// AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
// We exclude the entries from [2.3]
{ruleRTL, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN | 1<<bidi.NSM},
},
ruleRTLFinal: {
// [2.3] In an RTL label, the end of the label must be a character with
// Bidi property R, AL, EN, or AN, followed by zero or more characters
// with Bidi property NSM.
{ruleRTLFinal, 1<<bidi.R | 1<<bidi.AL | 1<<bidi.EN | 1<<bidi.AN | 1<<bidi.NSM},
// [2.2] In an RTL label, only characters with the Bidi properties R,
// AL, AN, EN, ES, CS, ET, ON, BN, or NSM are allowed.
// We exclude the entries from [2.3] and NSM.
{ruleRTL, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN},
},
ruleLTR: {
// [2.6] In an LTR label, the end of the label must be a character with
// Bidi property L or EN, followed by zero or more characters with Bidi
// property NSM.
{ruleLTRFinal, 1<<bidi.L | 1<<bidi.EN},
// [2.5] In an LTR label, only characters with the Bidi properties L,
// EN, ES, CS, ET, ON, BN, or NSM are allowed.
// We exclude the entries from [2.6].
{ruleLTR, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN | 1<<bidi.NSM},
},
ruleLTRFinal: {
// [2.6] In an LTR label, the end of the label must be a character with
// Bidi property L or EN, followed by zero or more characters with Bidi
// property NSM.
{ruleLTRFinal, 1<<bidi.L | 1<<bidi.EN | 1<<bidi.NSM},
// [2.5] In an LTR label, only characters with the Bidi properties L,
// EN, ES, CS, ET, ON, BN, or NSM are allowed.
// We exclude the entries from [2.6].
{ruleLTR, 1<<bidi.ES | 1<<bidi.CS | 1<<bidi.ET | 1<<bidi.ON | 1<<bidi.BN},
},
ruleInvalid: {
{ruleInvalid, 0},
{ruleInvalid, 0},
},
}
// [2.4] In an RTL label, if an EN is present, no AN may be present, and
// vice versa.
const exclusiveRTL = uint16(1<<bidi.EN | 1<<bidi.AN)
// From RFC 5893
// An RTL label is a label that contains at least one character of type
// R, AL, or AN.
//
// An LTR label is any label that is not an RTL label.
// Direction reports the direction of the given label as defined by RFC 5893.
// The Bidi Rule does not have to be applied to labels of the category
// LeftToRight.
func Direction(b []byte) bidi.Direction {
for i := 0; i < len(b); {
e, sz := bidi.Lookup(b[i:])
if sz == 0 {
i++
}
c := e.Class()
if c == bidi.R || c == bidi.AL || c == bidi.AN {
return bidi.RightToLeft
}
i += sz
}
return bidi.LeftToRight
}
// DirectionString reports the direction of the given label as defined by RFC
// 5893. The Bidi Rule does not have to be applied to labels of the category
// LeftToRight.
func DirectionString(s string) bidi.Direction {
for i := 0; i < len(s); {
e, sz := bidi.LookupString(s[i:])
if sz == 0 {
i++
continue
}
c := e.Class()
if c == bidi.R || c == bidi.AL || c == bidi.AN {
return bidi.RightToLeft
}
i += sz
}
return bidi.LeftToRight
}
// Valid reports whether b conforms to the BiDi rule.
func Valid(b []byte) bool {
var t Transformer
if n, ok := t.advance(b); !ok || n < len(b) {
return false
}
return t.isFinal()
}
// ValidString reports whether s conforms to the BiDi rule.
func ValidString(s string) bool {
var t Transformer
if n, ok := t.advanceString(s); !ok || n < len(s) {
return false
}
return t.isFinal()
}
// New returns a Transformer that verifies that input adheres to the Bidi Rule.
func New() *Transformer {
return &Transformer{}
}
// Transformer implements transform.Transform.
type Transformer struct {
state ruleState
hasRTL bool
seen uint16
}
// A rule can only be violated for "Bidi Domain names", meaning if one of the
// following categories has been observed.
func (t *Transformer) isRTL() bool {
const isRTL = 1<<bidi.R | 1<<bidi.AL | 1<<bidi.AN
return t.seen&isRTL != 0
}
// Reset implements transform.Transformer.
func (t *Transformer) Reset() { *t = Transformer{} }
// Transform implements transform.Transformer. This Transformer has state and
// needs to be reset between uses.
func (t *Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
if len(dst) < len(src) {
src = src[:len(dst)]
atEOF = false
err = transform.ErrShortDst
}
n, err1 := t.Span(src, atEOF)
copy(dst, src[:n])
if err == nil || err1 != nil && err1 != transform.ErrShortSrc {
err = err1
}
return n, n, err
}
// Span returns the first n bytes of src that conform to the Bidi rule.
func (t *Transformer) Span(src []byte, atEOF bool) (n int, err error) {
if t.state == ruleInvalid && t.isRTL() {
return 0, ErrInvalid
}
n, ok := t.advance(src)
switch {
case !ok:
err = ErrInvalid
case n < len(src):
if !atEOF {
err = transform.ErrShortSrc
break
}
err = ErrInvalid
case !t.isFinal():
err = ErrInvalid
}
return n, err
}
// Precomputing the ASCII values decreases running time for the ASCII fast path
// by about 30%.
var asciiTable [128]bidi.Properties
func init() {
for i := range asciiTable {
p, _ := bidi.LookupRune(rune(i))
asciiTable[i] = p
}
}
func (t *Transformer) advance(s []byte) (n int, ok bool) {
var e bidi.Properties
var sz int
for n < len(s) {
if s[n] < utf8.RuneSelf {
e, sz = asciiTable[s[n]], 1
} else {
e, sz = bidi.Lookup(s[n:])
if sz <= 1 {
if sz == 1 {
// We always consider invalid UTF-8 to be invalid, even if
// the string has not yet been determined to be RTL.
// TODO: is this correct?
return n, false
}
return n, true // incomplete UTF-8 encoding
}
}
// TODO: using CompactClass would result in noticeable speedup.
// See unicode/bidi/prop.go:Properties.CompactClass.
c := uint16(1 << e.Class())
t.seen |= c
if t.seen&exclusiveRTL == exclusiveRTL {
t.state = ruleInvalid
return n, false
}
switch tr := transitions[t.state]; {
case tr[0].mask&c != 0:
t.state = tr[0].next
case tr[1].mask&c != 0:
t.state = tr[1].next
default:
t.state = ruleInvalid
if t.isRTL() {
return n, false
}
}
n += sz
}
return n, true
}
func (t *Transformer) advanceString(s string) (n int, ok bool) {
var e bidi.Properties
var sz int
for n < len(s) {
if s[n] < utf8.RuneSelf {
e, sz = asciiTable[s[n]], 1
} else {
e, sz = bidi.LookupString(s[n:])
if sz <= 1 {
if sz == 1 {
return n, false // invalid UTF-8
}
return n, true // incomplete UTF-8 encoding
}
}
// TODO: using CompactClass results in noticeable speedup.
// See unicode/bidi/prop.go:Properties.CompactClass.
c := uint16(1 << e.Class())
t.seen |= c
if t.seen&exclusiveRTL == exclusiveRTL {
t.state = ruleInvalid
return n, false
}
switch tr := transitions[t.state]; {
case tr[0].mask&c != 0:
t.state = tr[0].next
case tr[1].mask&c != 0:
t.state = tr[1].next
default:
t.state = ruleInvalid
if t.isRTL() {
return n, false
}
}
n += sz
}
return n, true
}
vendor/golang.org/x/text/secure/bidirule/bidirule10.0.0.go 0 → 100644
View file @ 5bee5d69
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.10
package bidirule
func (t *Transformer) isFinal() bool {
return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial
}
vendor/golang.org/x/text/secure/bidirule/bidirule9.0.0.go 0 → 100644
View file @ 5bee5d69
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !go1.10
package bidirule
func (t *Transformer) isFinal() bool {
if !t.isRTL() {
return true
}
return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial
}
vendor/golang.org/x/text/unicode/bidi/bidi.go 0 → 100644
View file @ 5bee5d69
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:generate go run gen.go gen_trieval.go gen_ranges.go
// Package bidi contains functionality for bidirectional text support.
//
// See http://www.unicode.org/reports/tr9.
//
// NOTE: UNDER CONSTRUCTION. This API may change in backwards incompatible ways
// and without notice.
package bidi // import "golang.org/x/text/unicode/bidi"
// TODO:
// The following functionality would not be hard to implement, but hinges on
// the definition of a Segmenter interface. For now this is up to the user.
// - Iterate over paragraphs
// - Segmenter to iterate over runs directly from a given text.
// Also:
// - Transformer for reordering?
// - Transformer (validator, really) for Bidi Rule.
// This API tries to avoid dealing with embedding levels for now. Under the hood
// these will be computed, but the question is to which extent the user should
// know they exist. We should at some point allow the user to specify an
// embedding hierarchy, though.
// A Direction indicates the overall flow of text.
type Direction int
const (
// LeftToRight indicates the text contains no right-to-left characters and
// that either there are some left-to-right characters or the option
// DefaultDirection(LeftToRight) was passed.
LeftToRight Direction = iota
// RightToLeft indicates the text contains no left-to-right characters and
// that either there are some right-to-left characters or the option
// DefaultDirection(RightToLeft) was passed.
RightToLeft
// Mixed indicates text contains both left-to-right and right-to-left
// characters.
Mixed
// Neutral means that text contains no left-to-right and right-to-left
// characters and that no default direction has been set.
Neutral
)
type options struct{}
// An Option is an option for Bidi processing.
type Option func(*options)
// ICU allows the user to define embedding levels. This may be used, for example,
// to use hierarchical structure of markup languages to define embeddings.
// The following option may be a way to expose this functionality in this API.
// // LevelFunc sets a function that associates nesting levels with the given text.
// // The levels function will be called with monotonically increasing values for p.
// func LevelFunc(levels func(p int) int) Option {
// panic("unimplemented")
// }
// DefaultDirection sets the default direction for a Paragraph. The direction is
// overridden if the text contains directional characters.
func DefaultDirection(d Direction) Option {
panic("unimplemented")
}
// A Paragraph holds a single Paragraph for Bidi processing.
type Paragraph struct {
// buffers
}
// SetBytes configures p for the given paragraph text. It replaces text
// previously set by SetBytes or SetString. If b contains a paragraph separator
// it will only process the first paragraph and report the number of bytes
// consumed from b including this separator. Error may be non-nil if options are
// given.
func (p *Paragraph) SetBytes(b []byte, opts ...Option) (n int, err error) {
panic("unimplemented")
}
// SetString configures p for the given paragraph text. It replaces text
// previously set by SetBytes or SetString. If b contains a paragraph separator
// it will only process the first paragraph and report the number of bytes
// consumed from b including this separator. Error may be non-nil if options are
// given.
func (p *Paragraph) SetString(s string, opts ...Option) (n int, err error) {
panic("unimplemented")
}
// IsLeftToRight reports whether the principle direction of rendering for this
// paragraphs is left-to-right. If this returns false, the principle direction
// of rendering is right-to-left.
func (p *Paragraph) IsLeftToRight() bool {
panic("unimplemented")
}
// Direction returns the direction of the text of this paragraph.
//
// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
func (p *Paragraph) Direction() Direction {
panic("unimplemented")
}
// RunAt reports the Run at the given position of the input text.
//
// This method can be used for computing line breaks on paragraphs.
func (p *Paragraph) RunAt(pos int) Run {
panic("unimplemented")
}
// Order computes the visual ordering of all the runs in a Paragraph.
func (p *Paragraph) Order() (Ordering, error) {
panic("unimplemented")
}
// Line computes the visual ordering of runs for a single line starting and
// ending at the given positions in the original text.
func (p *Paragraph) Line(start, end int) (Ordering, error) {
panic("unimplemented")
}
// An Ordering holds the computed visual order of runs of a Paragraph. Calling
// SetBytes or SetString on the originating Paragraph invalidates an Ordering.
// The methods of an Ordering should only be called by one goroutine at a time.
type Ordering struct{}
// Direction reports the directionality of the runs.
//
// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
func (o *Ordering) Direction() Direction {
panic("unimplemented")
}
// NumRuns returns the number of runs.
func (o *Ordering) NumRuns() int {
panic("unimplemented")
}
// Run returns the ith run within the ordering.
func (o *Ordering) Run(i int) Run {
panic("unimplemented")
}
// TODO: perhaps with options.
// // Reorder creates a reader that reads the runes in visual order per character.
// // Modifiers remain after the runes they modify.
// func (l *Runs) Reorder() io.Reader {
// panic("unimplemented")
// }
// A Run is a continuous sequence of characters of a single direction.
type Run struct {
}
// String returns the text of the run in its original order.
func (r *Run) String() string {
panic("unimplemented")
}
// Bytes returns the text of the run in its original order.
func (r *Run) Bytes() []byte {
panic("unimplemented")
}
// TODO: methods for
// - Display order
// - headers and footers
// - bracket replacement.
// Direction reports the direction of the run.
func (r *Run) Direction() Direction {
panic("unimplemented")
}
// Position of the Run within the text passed to SetBytes or SetString of the
// originating Paragraph value.
func (r *Run) Pos() (start, end int) {
panic("unimplemented")
}
// AppendReverse reverses the order of characters of in, appends them to out,
// and returns the result. Modifiers will still follow the runes they modify.
// Brackets are replaced with their counterparts.
func AppendReverse(out, in []byte) []byte {
panic("unimplemented")
}
// ReverseString reverses the order of characters in s and returns a new string.
// Modifiers will still follow the runes they modify. Brackets are replaced with
// their counterparts.
func ReverseString(s string) string {
panic("unimplemented")
}
vendor/golang.org/x/text/unicode/bidi/bracket.go 0 → 100644
View file @ 5bee5d69
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bidi
import (
"container/list"
"fmt"
"sort"
)
// This file contains a port of the reference implementation of the
// Bidi Parentheses Algorithm:
// http://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/BidiPBAReference.java
//
// The implementation in this file covers definitions BD14-BD16 and rule N0
// of UAX#9.
//
// Some preprocessing is done for each rune before data is passed to this
// algorithm:
// - opening and closing brackets are identified
// - a bracket pair type, like '(' and ')' is assigned a unique identifier that
// is identical for the opening and closing bracket. It is left to do these
// mappings.
// - The BPA algorithm requires that bracket characters that are canonical
// equivalents of each other be able to be substituted for each other.
// It is the responsibility of the caller to do this canonicalization.
//
// In implementing BD16, this implementation departs slightly from the "logical"
// algorithm defined in UAX#9. In particular, the stack referenced there
// supports operations that go beyond a "basic" stack. An equivalent
// implementation based on a linked list is used here.
// Bidi_Paired_Bracket_Type
// BD14. An opening paired bracket is a character whose
// Bidi_Paired_Bracket_Type property value is Open.
//
// BD15. A closing paired bracket is a character whose
// Bidi_Paired_Bracket_Type property value is Close.
type bracketType byte
const (
bpNone bracketType = iota
bpOpen
bpClose
)
// bracketPair holds a pair of index values for opening and closing bracket
// location of a bracket pair.
type bracketPair struct {
opener int
closer int
}
func (b *bracketPair) String() string {
return fmt.Sprintf("(%v, %v)", b.opener, b.closer)
}
// bracketPairs is a slice of bracketPairs with a sort.Interface implementation.
type bracketPairs []bracketPair
func (b bracketPairs) Len() int { return len(b) }
func (b bracketPairs) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b bracketPairs) Less(i, j int) bool { return b[i].opener < b[j].opener }
// resolvePairedBrackets runs the paired bracket part of the UBA algorithm.
//
// For each rune, it takes the indexes into the original string, the class the
// bracket type (in pairTypes) and the bracket identifier (pairValues). It also
// takes the direction type for the start-of-sentence and the embedding level.
//
// The identifiers for bracket types are the rune of the canonicalized opening
// bracket for brackets (open or close) or 0 for runes that are not brackets.
func resolvePairedBrackets(s *isolatingRunSequence) {
p := bracketPairer{
sos: s.sos,
openers: list.New(),
codesIsolatedRun: s.types,
indexes: s.indexes,
}
dirEmbed := L
if s.level&1 != 0 {
dirEmbed = R
}
p.locateBrackets(s.p.pairTypes, s.p.pairValues)
p.resolveBrackets(dirEmbed, s.p.initialTypes)
}
type bracketPairer struct {
sos Class // direction corresponding to start of sequence
// The following is a restatement of BD 16 using non-algorithmic language.
//
// A bracket pair is a pair of characters consisting of an opening
// paired bracket and a closing paired bracket such that the
// Bidi_Paired_Bracket property value of the former equals the latter,
// subject to the following constraints.
// - both characters of a pair occur in the same isolating run sequence
// - the closing character of a pair follows the opening character
// - any bracket character can belong at most to one pair, the earliest possible one
// - any bracket character not part of a pair is treated like an ordinary character
// - pairs may nest properly, but their spans may not overlap otherwise
// Bracket characters with canonical decompositions are supposed to be
// treated as if they had been normalized, to allow normalized and non-
// normalized text to give the same result. In this implementation that step
// is pushed out to the caller. The caller has to ensure that the pairValue
// slices contain the rune of the opening bracket after normalization for
// any opening or closing bracket.
openers *list.List // list of positions for opening brackets
// bracket pair positions sorted by location of opening bracket
pairPositions bracketPairs
codesIsolatedRun []Class // directional bidi codes for an isolated run
indexes []int // array of index values into the original string
}
// matchOpener reports whether characters at given positions form a matching
// bracket pair.
func (p *bracketPairer) matchOpener(pairValues []rune, opener, closer int) bool {
return pairValues[p.indexes[opener]] == pairValues[p.indexes[closer]]
}
const maxPairingDepth = 63
// locateBrackets locates matching bracket pairs according to BD16.
//
// This implementation uses a linked list instead of a stack, because, while
// elements are added at the front (like a push) they are not generally removed
// in atomic 'pop' operations, reducing the benefit of the stack archetype.
func (p *bracketPairer) locateBrackets(pairTypes []bracketType, pairValues []rune) {
// traverse the run
// do that explicitly (not in a for-each) so we can record position
for i, index := range p.indexes {
// look at the bracket type for each character
if pairTypes[index] == bpNone || p.codesIsolatedRun[i] != ON {
// continue scanning
continue
}
switch pairTypes[index] {
case bpOpen:
// check if maximum pairing depth reached
if p.openers.Len() == maxPairingDepth {
p.openers.Init()
return
}
// remember opener location, most recent first
p.openers.PushFront(i)
case bpClose:
// see if there is a match
count := 0
for elem := p.openers.Front(); elem != nil; elem = elem.Next() {
count++
opener := elem.Value.(int)
if p.matchOpener(pairValues, opener, i) {
// if the opener matches, add nested pair to the ordered list
p.pairPositions = append(p.pairPositions, bracketPair{opener, i})
// remove up to and including matched opener
for ; count > 0; count-- {
p.openers.Remove(p.openers.Front())
}
break
}
}
sort.Sort(p.pairPositions)
// if we get here, the closing bracket matched no openers
// and gets ignored
}
}
}
// Bracket pairs within an isolating run sequence are processed as units so
// that both the opening and the closing paired bracket in a pair resolve to
// the same direction.
//
// N0. Process bracket pairs in an isolating run sequence sequentially in
// the logical order of the text positions of the opening paired brackets
// using the logic given below. Within this scope, bidirectional types EN
// and AN are treated as R.
//
// Identify the bracket pairs in the current isolating run sequence
// according to BD16. For each bracket-pair element in the list of pairs of
// text positions:
//
// a Inspect the bidirectional types of the characters enclosed within the
// bracket pair.
//
// b If any strong type (either L or R) matching the embedding direction is
// found, set the type for both brackets in the pair to match the embedding
// direction.
//
// o [ e ] o -> o e e e o
//
// o [ o e ] -> o e o e e
//
// o [ NI e ] -> o e NI e e
//
// c Otherwise, if a strong type (opposite the embedding direction) is
// found, test for adjacent strong types as follows: 1 First, check
// backwards before the opening paired bracket until the first strong type
// (L, R, or sos) is found. If that first preceding strong type is opposite
// the embedding direction, then set the type for both brackets in the pair
// to that type. 2 Otherwise, set the type for both brackets in the pair to
// the embedding direction.
//
// o [ o ] e -> o o o o e
//
// o [ o NI ] o -> o o o NI o o
//
// e [ o ] o -> e e o e o
//
// e [ o ] e -> e e o e e
//
// e ( o [ o ] NI ) e -> e e o o o o NI e e
//
// d Otherwise, do not set the type for the current bracket pair. Note that
// if the enclosed text contains no strong types the paired brackets will
// both resolve to the same level when resolved individually using rules N1
// and N2.
//
// e ( NI ) o -> e ( NI ) o
// getStrongTypeN0 maps character's directional code to strong type as required
// by rule N0.
//
// TODO: have separate type for "strong" directionality.
func (p *bracketPairer) getStrongTypeN0(index int) Class {
switch p.codesIsolatedRun[index] {
// in the scope of N0, number types are treated as R
case EN, AN, AL, R:
return R
case L:
return L
default:
return ON
}
}
// classifyPairContent reports the strong types contained inside a Bracket Pair,
// assuming the given embedding direction.
//
// It returns ON if no strong type is found. If a single strong type is found,
// it returns this this type. Otherwise it returns the embedding direction.
//
// TODO: use separate type for "strong" directionality.
func (p *bracketPairer) classifyPairContent(loc bracketPair, dirEmbed Class) Class {
dirOpposite := ON
for i := loc.opener + 1; i < loc.closer; i++ {
dir := p.getStrongTypeN0(i)
if dir == ON {
continue
}
if dir == dirEmbed {
return dir // type matching embedding direction found
}
dirOpposite = dir
}
// return ON if no strong type found, or class opposite to dirEmbed
return dirOpposite
}
// classBeforePair determines which strong types are present before a Bracket
// Pair. Return R or L if strong type found, otherwise ON.
func (p *bracketPairer) classBeforePair(loc bracketPair) Class {
for i := loc.opener - 1; i >= 0; i-- {
if dir := p.getStrongTypeN0(i); dir != ON {
return dir
}
}
// no strong types found, return sos
return p.sos
}
// assignBracketType implements rule N0 for a single bracket pair.
func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class, initialTypes []Class) {
// rule "N0, a", inspect contents of pair
dirPair := p.classifyPairContent(loc, dirEmbed)
// dirPair is now L, R, or N (no strong type found)
// the following logical tests are performed out of order compared to
// the statement of the rules but yield the same results
if dirPair == ON {
return // case "d" - nothing to do
}
if dirPair != dirEmbed {
// case "c": strong type found, opposite - check before (c.1)
dirPair = p.classBeforePair(loc)
if dirPair == dirEmbed || dirPair == ON {
// no strong opposite type found before - use embedding (c.2)
dirPair = dirEmbed
}
}
// else: case "b", strong type found matching embedding,
// no explicit action needed, as dirPair is already set to embedding
// direction
// set the bracket types to the type found
p.setBracketsToType(loc, dirPair, initialTypes)
}
func (p *bracketPairer) setBracketsToType(loc bracketPair, dirPair Class, initialTypes []Class) {
p.codesIsolatedRun[loc.opener] = dirPair
p.codesIsolatedRun[loc.closer] = dirPair
for i := loc.opener + 1; i < loc.closer; i++ {
index := p.indexes[i]
if initialTypes[index] != NSM {
break
}
p.codesIsolatedRun[i] = dirPair
}
for i := loc.closer + 1; i < len(p.indexes); i++ {
index := p.indexes[i]
if initialTypes[index] != NSM {
break
}
p.codesIsolatedRun[i] = dirPair
}
}
// resolveBrackets implements rule N0 for a list of pairs.
func (p *bracketPairer) resolveBrackets(dirEmbed Class, initialTypes []Class) {
for _, loc := range p.pairPositions {
p.assignBracketType(loc, dirEmbed, initialTypes)
}
}
vendor/golang.org/x/text/unicode/bidi/core.go 0 → 100644
View file @ 5bee5d69
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bidi
import "log"
// This implementation is a port based on the reference implementation found at:
// http://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/
//
// described in Unicode Bidirectional Algorithm (UAX #9).
//
// Input:
// There are two levels of input to the algorithm, since clients may prefer to
// supply some information from out-of-band sources rather than relying on the
// default behavior.
//
// - Bidi class array
// - Bidi class array, with externally supplied base line direction
//
// Output:
// Output is separated into several stages:
//
// - levels array over entire paragraph
// - reordering array over entire paragraph
// - levels array over line
// - reordering array over line
//
// Note that for conformance to the Unicode Bidirectional Algorithm,
// implementations are only required to generate correct reordering and
// character directionality (odd or even levels) over a line. Generating
// identical level arrays over a line is not required. Bidi explicit format
// codes (LRE, RLE, LRO, RLO, PDF) and BN can be assigned arbitrary levels and
// positions as long as the rest of the input is properly reordered.
//
// As the algorithm is defined to operate on a single paragraph at a time, this
// implementation is written to handle single paragraphs. Thus rule P1 is
// presumed by this implementation-- the data provided to the implementation is
// assumed to be a single paragraph, and either contains no 'B' codes, or a
// single 'B' code at the end of the input. 'B' is allowed as input to
// illustrate how the algorithm assigns it a level.
//
// Also note that rules L3 and L4 depend on the rendering engine that uses the
// result of the bidi algorithm. This implementation assumes that the rendering
// engine expects combining marks in visual order (e.g. to the left of their
// base character in RTL runs) and that it adjusts the glyphs used to render
// mirrored characters that are in RTL runs so that they render appropriately.
// level is the embedding level of a character. Even embedding levels indicate
// left-to-right order and odd levels indicate right-to-left order. The special
// level of -1 is reserved for undefined order.
type level int8
const implicitLevel level = -1
// in returns if x is equal to any of the values in set.
func (c Class) in(set ...Class) bool {
for _, s := range set {
if c == s {
return true
}
}
return false
}
// A paragraph contains the state of a paragraph.
type paragraph struct {
initialTypes []Class
// Arrays of properties needed for paired bracket evaluation in N0
pairTypes []bracketType // paired Bracket types for paragraph
pairValues []rune // rune for opening bracket or pbOpen and pbClose; 0 for pbNone
embeddingLevel level // default: = implicitLevel;
// at the paragraph levels
resultTypes []Class
resultLevels []level
// Index of matching PDI for isolate initiator characters. For other
// characters, the value of matchingPDI will be set to -1. For isolate
// initiators with no matching PDI, matchingPDI will be set to the length of
// the input string.
matchingPDI []int
// Index of matching isolate initiator for PDI characters. For other
// characters, and for PDIs with no matching isolate initiator, the value of
// matchingIsolateInitiator will be set to -1.
matchingIsolateInitiator []int
}
// newParagraph initializes a paragraph. The user needs to supply a few arrays
// corresponding to the preprocessed text input. The types correspond to the
// Unicode BiDi classes for each rune. pairTypes indicates the bracket type for
// each rune. pairValues provides a unique bracket class identifier for each
// rune (suggested is the rune of the open bracket for opening and matching
// close brackets, after normalization). The embedding levels are optional, but
// may be supplied to encode embedding levels of styled text.
//
// TODO: return an error.
func newParagraph(types []Class, pairTypes []bracketType, pairValues []rune, levels level) *paragraph {
validateTypes(types)
validatePbTypes(pairTypes)
validatePbValues(pairValues, pairTypes)
validateParagraphEmbeddingLevel(levels)
p := &paragraph{
initialTypes: append([]Class(nil), types...),
embeddingLevel: levels,
pairTypes: pairTypes,
pairValues: pairValues,
resultTypes: append([]Class(nil), types...),
}
p.run()
return p
}
func (p *paragraph) Len() int { return len(p.initialTypes) }
// The algorithm. Does not include line-based processing (Rules L1, L2).
// These are applied later in the line-based phase of the algorithm.
func (p *paragraph) run() {
p.determineMatchingIsolates()
// 1) determining the paragraph level
// Rule P1 is the requirement for entering this algorithm.
// Rules P2, P3.
// If no externally supplied paragraph embedding level, use default.
if p.embeddingLevel == implicitLevel {
p.embeddingLevel = p.determineParagraphEmbeddingLevel(0, p.Len())
}
// Initialize result levels to paragraph embedding level.
p.resultLevels = make([]level, p.Len())
setLevels(p.resultLevels, p.embeddingLevel)
// 2) Explicit levels and directions
// Rules X1-X8.
p.determineExplicitEmbeddingLevels()
// Rule X9.
// We do not remove the embeddings, the overrides, the PDFs, and the BNs
// from the string explicitly. But they are not copied into isolating run
// sequences when they are created, so they are removed for all
// practical purposes.
// Rule X10.
// Run remainder of algorithm one isolating run sequence at a time
for _, seq := range p.determineIsolatingRunSequences() {
// 3) resolving weak types
// Rules W1-W7.
seq.resolveWeakTypes()
// 4a) resolving paired brackets
// Rule N0
resolvePairedBrackets(seq)
// 4b) resolving neutral types
// Rules N1-N3.
seq.resolveNeutralTypes()
// 5) resolving implicit embedding levels
// Rules I1, I2.
seq.resolveImplicitLevels()
// Apply the computed levels and types
seq.applyLevelsAndTypes()
}
// Assign appropriate levels to 'hide' LREs, RLEs, LROs, RLOs, PDFs, and
// BNs. This is for convenience, so the resulting level array will have
// a value for every character.
p.assignLevelsToCharactersRemovedByX9()
}
// determineMatchingIsolates determines the matching PDI for each isolate
// initiator and vice versa.
//
// Definition BD9.
//
// At the end of this function:
//
// - The member variable matchingPDI is set to point to the index of the
// matching PDI character for each isolate initiator character. If there is
// no matching PDI, it is set to the length of the input text. For other
// characters, it is set to -1.
// - The member variable matchingIsolateInitiator is set to point to the
// index of the matching isolate initiator character for each PDI character.
// If there is no matching isolate initiator, or the character is not a PDI,
// it is set to -1.
func (p *paragraph) determineMatchingIsolates() {
p.matchingPDI = make([]int, p.Len())
p.matchingIsolateInitiator = make([]int, p.Len())
for i := range p.matchingIsolateInitiator {
p.matchingIsolateInitiator[i] = -1
}
for i := range p.matchingPDI {
p.matchingPDI[i] = -1
if t := p.resultTypes[i]; t.in(LRI, RLI, FSI) {
depthCounter := 1
for j := i + 1; j < p.Len(); j++ {
if u := p.resultTypes[j]; u.in(LRI, RLI, FSI) {
depthCounter++
} else if u == PDI {
if depthCounter--; depthCounter == 0 {
p.matchingPDI[i] = j
p.matchingIsolateInitiator[j] = i
break
}
}
}
if p.matchingPDI[i] == -1 {
p.matchingPDI[i] = p.Len()
}
}
}
}
// determineParagraphEmbeddingLevel reports the resolved paragraph direction of
// the substring limited by the given range [start, end).
//
// Determines the paragraph level based on rules P2, P3. This is also used
// in rule X5c to find if an FSI should resolve to LRI or RLI.
func (p *paragraph) determineParagraphEmbeddingLevel(start, end int) level {
var strongType Class = unknownClass
// Rule P2.
for i := start; i < end; i++ {
if t := p.resultTypes[i]; t.in(L, AL, R) {
strongType = t
break
} else if t.in(FSI, LRI, RLI) {
i = p.matchingPDI[i] // skip over to the matching PDI
if i > end {
log.Panic("assert (i <= end)")
}
}
}
// Rule P3.
switch strongType {
case unknownClass: // none found
// default embedding level when no strong types found is 0.
return 0
case L:
return 0
default: // AL, R
return 1
}
}
const maxDepth = 125
// This stack will store the embedding levels and override and isolated
// statuses
type directionalStatusStack struct {
stackCounter int
embeddingLevelStack [maxDepth + 1]level
overrideStatusStack [maxDepth + 1]Class
isolateStatusStack [maxDepth + 1]bool
}
func (s *directionalStatusStack) empty() { s.stackCounter = 0 }
func (s *directionalStatusStack) pop() { s.stackCounter-- }
func (s *directionalStatusStack) depth() int { return s.stackCounter }
func (s *directionalStatusStack) push(level level, overrideStatus Class, isolateStatus bool) {
s.embeddingLevelStack[s.stackCounter] = level
s.overrideStatusStack[s.stackCounter] = overrideStatus
s.isolateStatusStack[s.stackCounter] = isolateStatus
s.stackCounter++
}
func (s *directionalStatusStack) lastEmbeddingLevel() level {
return s.embeddingLevelStack[s.stackCounter-1]
}
func (s *directionalStatusStack) lastDirectionalOverrideStatus() Class {
return s.overrideStatusStack[s.stackCounter-1]
}
func (s *directionalStatusStack) lastDirectionalIsolateStatus() bool {
return s.isolateStatusStack[s.stackCounter-1]
}
// Determine explicit levels using rules X1 - X8
func (p *paragraph) determineExplicitEmbeddingLevels() {
var stack directionalStatusStack
var overflowIsolateCount, overflowEmbeddingCount, validIsolateCount int
// Rule X1.
stack.push(p.embeddingLevel, ON, false)
for i, t := range p.resultTypes {
// Rules X2, X3, X4, X5, X5a, X5b, X5c
switch t {
case RLE, LRE, RLO, LRO, RLI, LRI, FSI:
isIsolate := t.in(RLI, LRI, FSI)
isRTL := t.in(RLE, RLO, RLI)
// override if this is an FSI that resolves to RLI
if t == FSI {
isRTL = (p.determineParagraphEmbeddingLevel(i+1, p.matchingPDI[i]) == 1)
}
if isIsolate {
p.resultLevels[i] = stack.lastEmbeddingLevel()
if stack.lastDirectionalOverrideStatus() != ON {
p.resultTypes[i] = stack.lastDirectionalOverrideStatus()
}
}
var newLevel level
if isRTL {
// least greater odd
newLevel = (stack.lastEmbeddingLevel() + 1) | 1
} else {
// least greater even
newLevel = (stack.lastEmbeddingLevel() + 2) &^ 1
}
if newLevel <= maxDepth && overflowIsolateCount == 0 && overflowEmbeddingCount == 0 {
if isIsolate {
validIsolateCount++
}
// Push new embedding level, override status, and isolated
// status.
// No check for valid stack counter, since the level check
// suffices.
switch t {
case LRO:
stack.push(newLevel, L, isIsolate)
case RLO:
stack.push(newLevel, R, isIsolate)
default:
stack.push(newLevel, ON, isIsolate)
}
// Not really part of the spec
if !isIsolate {
p.resultLevels[i] = newLevel
}
} else {
// This is an invalid explicit formatting character,
// so apply the "Otherwise" part of rules X2-X5b.
if isIsolate {
overflowIsolateCount++
} else { // !isIsolate
if overflowIsolateCount == 0 {
overflowEmbeddingCount++
}
}
}
// Rule X6a
case PDI:
if overflowIsolateCount > 0 {
overflowIsolateCount--
} else if validIsolateCount == 0 {
// do nothing
} else {
overflowEmbeddingCount = 0
for !stack.lastDirectionalIsolateStatus() {
stack.pop()
}
stack.pop()
validIsolateCount--
}
p.resultLevels[i] = stack.lastEmbeddingLevel()
// Rule X7
case PDF:
// Not really part of the spec
p.resultLevels[i] = stack.lastEmbeddingLevel()
if overflowIsolateCount > 0 {
// do nothing
} else if overflowEmbeddingCount > 0 {
overflowEmbeddingCount--
} else if !stack.lastDirectionalIsolateStatus() && stack.depth() >= 2 {
stack.pop()
}
case B: // paragraph separator.
// Rule X8.
// These values are reset for clarity, in this implementation B
// can only occur as the last code in the array.
stack.empty()
overflowIsolateCount = 0
overflowEmbeddingCount = 0
validIsolateCount = 0
p.resultLevels[i] = p.embeddingLevel
default:
p.resultLevels[i] = stack.lastEmbeddingLevel()
if stack.lastDirectionalOverrideStatus() != ON {
p.resultTypes[i] = stack.lastDirectionalOverrideStatus()
}
}
}
}
type isolatingRunSequence struct {
p *paragraph
indexes []int // indexes to the original string
types []Class // type of each character using the index
resolvedLevels []level // resolved levels after application of rules
level level
sos, eos Class
}
func (i *isolatingRunSequence) Len() int { return len(i.indexes) }
func maxLevel(a, b level) level {
if a > b {
return a
}
return b
}
// Rule X10, second bullet: Determine the start-of-sequence (sos) and end-of-sequence (eos) types,
// either L or R, for each isolating run sequence.
func (p *paragraph) isolatingRunSequence(indexes []int) *isolatingRunSequence {
length := len(indexes)
types := make([]Class, length)
for i, x := range indexes {
types[i] = p.resultTypes[x]
}
// assign level, sos and eos
prevChar := indexes[0] - 1
for prevChar >= 0 && isRemovedByX9(p.initialTypes[prevChar]) {
prevChar--
}
prevLevel := p.embeddingLevel
if prevChar >= 0 {
prevLevel = p.resultLevels[prevChar]
}
var succLevel level
lastType := types[length-1]
if lastType.in(LRI, RLI, FSI) {
succLevel = p.embeddingLevel
} else {
// the first character after the end of run sequence
limit := indexes[length-1] + 1
for ; limit < p.Len() && isRemovedByX9(p.initialTypes[limit]); limit++ {
}
succLevel = p.embeddingLevel
if limit < p.Len() {
succLevel = p.resultLevels[limit]
}
}
level := p.resultLevels[indexes[0]]
return &isolatingRunSequence{
p: p,
indexes: indexes,
types: types,
level: level,
sos: typeForLevel(maxLevel(prevLevel, level)),
eos: typeForLevel(maxLevel(succLevel, level)),
}
}
// Resolving weak types Rules W1-W7.
//
// Note that some weak types (EN, AN) remain after this processing is
// complete.
func (s *isolatingRunSequence) resolveWeakTypes() {
// on entry, only these types remain
s.assertOnly(L, R, AL, EN, ES, ET, AN, CS, B, S, WS, ON, NSM, LRI, RLI, FSI, PDI)
// Rule W1.
// Changes all NSMs.
preceedingCharacterType := s.sos
for i, t := range s.types {
if t == NSM {
s.types[i] = preceedingCharacterType
} else {
if t.in(LRI, RLI, FSI, PDI) {
preceedingCharacterType = ON
}
preceedingCharacterType = t
}
}
// Rule W2.
// EN does not change at the start of the run, because sos != AL.
for i, t := range s.types {
if t == EN {
for j := i - 1; j >= 0; j-- {
if t := s.types[j]; t.in(L, R, AL) {
if t == AL {
s.types[i] = AN
}
break
}
}
}
}
// Rule W3.
for i, t := range s.types {
if t == AL {
s.types[i] = R
}
}
// Rule W4.
// Since there must be values on both sides for this rule to have an
// effect, the scan skips the first and last value.
//
// Although the scan proceeds left to right, and changes the type
// values in a way that would appear to affect the computations
// later in the scan, there is actually no problem. A change in the
// current value can only affect the value to its immediate right,
// and only affect it if it is ES or CS. But the current value can
// only change if the value to its right is not ES or CS. Thus
// either the current value will not change, or its change will have
// no effect on the remainder of the analysis.
for i := 1; i < s.Len()-1; i++ {
t := s.types[i]
if t == ES || t == CS {
prevSepType := s.types[i-1]
succSepType := s.types[i+1]
if prevSepType == EN && succSepType == EN {
s.types[i] = EN
} else if s.types[i] == CS && prevSepType == AN && succSepType == AN {
s.types[i] = AN
}
}
}
// Rule W5.
for i, t := range s.types {
if t == ET {
// locate end of sequence
runStart := i
runEnd := s.findRunLimit(runStart, ET)
// check values at ends of sequence
t := s.sos
if runStart > 0 {
t = s.types[runStart-1]
}
if t != EN {
t = s.eos
if runEnd < len(s.types) {
t = s.types[runEnd]
}
}
if t == EN {
setTypes(s.types[runStart:runEnd], EN)
}
// continue at end of sequence
i = runEnd
}
}
// Rule W6.
for i, t := range s.types {
if t.in(ES, ET, CS) {
s.types[i] = ON
}
}
// Rule W7.
for i, t := range s.types {
if t == EN {
// set default if we reach start of run
prevStrongType := s.sos
for j := i - 1; j >= 0; j-- {
t = s.types[j]
if t == L || t == R { // AL's have been changed to R
prevStrongType = t
break
}
}
if prevStrongType == L {
s.types[i] = L
}
}
}
}
// 6) resolving neutral types Rules N1-N2.
func (s *isolatingRunSequence) resolveNeutralTypes() {
// on entry, only these types can be in resultTypes
s.assertOnly(L, R, EN, AN, B, S, WS, ON, RLI, LRI, FSI, PDI)
for i, t := range s.types {
switch t {
case WS, ON, B, S, RLI, LRI, FSI, PDI:
// find bounds of run of neutrals
runStart := i
runEnd := s.findRunLimit(runStart, B, S, WS, ON, RLI, LRI, FSI, PDI)
// determine effective types at ends of run
var leadType, trailType Class
// Note that the character found can only be L, R, AN, or
// EN.
if runStart == 0 {
leadType = s.sos
} else {
leadType = s.types[runStart-1]
if leadType.in(AN, EN) {
leadType = R
}
}
if runEnd == len(s.types) {
trailType = s.eos
} else {
trailType = s.types[runEnd]
if trailType.in(AN, EN) {
trailType = R
}
}
var resolvedType Class
if leadType == trailType {
// Rule N1.
resolvedType = leadType
} else {
// Rule N2.
// Notice the embedding level of the run is used, not
// the paragraph embedding level.
resolvedType = typeForLevel(s.level)
}
setTypes(s.types[runStart:runEnd], resolvedType)
// skip over run of (former) neutrals
i = runEnd
}
}
}
func setLevels(levels []level, newLevel level) {
for i := range levels {
levels[i] = newLevel
}
}
func setTypes(types []Class, newType Class) {
for i := range types {
types[i] = newType
}
}
// 7) resolving implicit embedding levels Rules I1, I2.
func (s *isolatingRunSequence) resolveImplicitLevels() {
// on entry, only these types can be in resultTypes
s.assertOnly(L, R, EN, AN)
s.resolvedLevels = make([]level, len(s.types))
setLevels(s.resolvedLevels, s.level)
if (s.level & 1) == 0 { // even level
for i, t := range s.types {
// Rule I1.
if t == L {
// no change
} else if t == R {
s.resolvedLevels[i] += 1
} else { // t == AN || t == EN
s.resolvedLevels[i] += 2
}
}
} else { // odd level
for i, t := range s.types {
// Rule I2.
if t == R {
// no change
} else { // t == L || t == AN || t == EN
s.resolvedLevels[i] += 1
}
}
}
}
// Applies the levels and types resolved in rules W1-I2 to the
// resultLevels array.
func (s *isolatingRunSequence) applyLevelsAndTypes() {
for i, x := range s.indexes {
s.p.resultTypes[x] = s.types[i]
s.p.resultLevels[x] = s.resolvedLevels[i]
}
}
// Return the limit of the run consisting only of the types in validSet
// starting at index. This checks the value at index, and will return
// index if that value is not in validSet.
func (s *isolatingRunSequence) findRunLimit(index int, validSet ...Class) int {
loop:
for ; index < len(s.types); index++ {
t := s.types[index]
for _, valid := range validSet {
if t == valid {
continue loop
}
}
return index // didn't find a match in validSet
}
return len(s.types)
}
// Algorithm validation. Assert that all values in types are in the
// provided set.
func (s *isolatingRunSequence) assertOnly(codes ...Class) {
loop:
for i, t := range s.types {
for _, c := range codes {
if t == c {
continue loop
}
}
log.Panicf("invalid bidi code %v present in assertOnly at position %d", t, s.indexes[i])
}
}
// determineLevelRuns returns an array of level runs. Each level run is
// described as an array of indexes into the input string.
//
// Determines the level runs. Rule X9 will be applied in determining the
// runs, in the way that makes sure the characters that are supposed to be
// removed are not included in the runs.
func (p *paragraph) determineLevelRuns() [][]int {
run := []int{}
allRuns := [][]int{}
currentLevel := implicitLevel
for i := range p.initialTypes {
if !isRemovedByX9(p.initialTypes[i]) {
if p.resultLevels[i] != currentLevel {
// we just encountered a new run; wrap up last run
if currentLevel >= 0 { // only wrap it up if there was a run
allRuns = append(allRuns, run)
run = nil
}
// Start new run
currentLevel = p.resultLevels[i]
}
run = append(run, i)
}
}
// Wrap up the final run, if any
if len(run) > 0 {
allRuns = append(allRuns, run)
}
return allRuns
}
// Definition BD13. Determine isolating run sequences.
func (p *paragraph) determineIsolatingRunSequences() []*isolatingRunSequence {
levelRuns := p.determineLevelRuns()
// Compute the run that each character belongs to
runForCharacter := make([]int, p.Len())
for i, run := range levelRuns {
for _, index := range run {
runForCharacter[index] = i
}
}
sequences := []*isolatingRunSequence{}
var currentRunSequence []int
for _, run := range levelRuns {
first := run[0]
if p.initialTypes[first] != PDI || p.matchingIsolateInitiator[first] == -1 {
currentRunSequence = nil
// int run = i;
for {
// Copy this level run into currentRunSequence
currentRunSequence = append(currentRunSequence, run...)
last := currentRunSequence[len(currentRunSequence)-1]
lastT := p.initialTypes[last]
if lastT.in(LRI, RLI, FSI) && p.matchingPDI[last] != p.Len() {
run = levelRuns[runForCharacter[p.matchingPDI[last]]]
} else {
break
}
}
sequences = append(sequences, p.isolatingRunSequence(currentRunSequence))
}
}
return sequences
}
// Assign level information to characters removed by rule X9. This is for
// ease of relating the level information to the original input data. Note
// that the levels assigned to these codes are arbitrary, they're chosen so
// as to avoid breaking level runs.
func (p *paragraph) assignLevelsToCharactersRemovedByX9() {
for i, t := range p.initialTypes {
if t.in(LRE, RLE, LRO, RLO, PDF, BN) {
p.resultTypes[i] = t
p.resultLevels[i] = -1
}
}
// now propagate forward the levels information (could have
// propagated backward, the main thing is not to introduce a level
// break where one doesn't already exist).
if p.resultLevels[0] == -1 {
p.resultLevels[0] = p.embeddingLevel
}
for i := 1; i < len(p.initialTypes); i++ {
if p.resultLevels[i] == -1 {
p.resultLevels[i] = p.resultLevels[i-1]
}
}
// Embedding information is for informational purposes only so need not be
// adjusted.
}
//
// Output
//
// getLevels computes levels array breaking lines at offsets in linebreaks.
// Rule L1.
//
// The linebreaks array must include at least one value. The values must be
// in strictly increasing order (no duplicates) between 1 and the length of
// the text, inclusive. The last value must be the length of the text.
func (p *paragraph) getLevels(linebreaks []int) []level {
// Note that since the previous processing has removed all
// P, S, and WS values from resultTypes, the values referred to
// in these rules are the initial types, before any processing
// has been applied (including processing of overrides).
//
// This example implementation has reinserted explicit format codes
// and BN, in order that the levels array correspond to the
// initial text. Their final placement is not normative.
// These codes are treated like WS in this implementation,
// so they don't interrupt sequences of WS.
validateLineBreaks(linebreaks, p.Len())
result := append([]level(nil), p.resultLevels...)
// don't worry about linebreaks since if there is a break within
// a series of WS values preceding S, the linebreak itself
// causes the reset.
for i, t := range p.initialTypes {
if t.in(B, S) {
// Rule L1, clauses one and two.
result[i] = p.embeddingLevel
// Rule L1, clause three.
for j := i - 1; j >= 0; j-- {
if isWhitespace(p.initialTypes[j]) { // including format codes
result[j] = p.embeddingLevel
} else {
break
}
}
}
}
// Rule L1, clause four.
start := 0
for _, limit := range linebreaks {
for j := limit - 1; j >= start; j-- {
if isWhitespace(p.initialTypes[j]) { // including format codes
result[j] = p.embeddingLevel
} else {
break
}
}
start = limit
}
return result
}
// getReordering returns the reordering of lines from a visual index to a
// logical index for line breaks at the given offsets.
//
// Lines are concatenated from left to right. So for example, the fifth
// character from the left on the third line is
//
// getReordering(linebreaks)[linebreaks[1] + 4]
//
// (linebreaks[1] is the position after the last character of the second
// line, which is also the index of the first character on the third line,
// and adding four gets the fifth character from the left).
//
// The linebreaks array must include at least one value. The values must be
// in strictly increasing order (no duplicates) between 1 and the length of
// the text, inclusive. The last value must be the length of the text.
func (p *paragraph) getReordering(linebreaks []int) []int {
validateLineBreaks(linebreaks, p.Len())
return computeMultilineReordering(p.getLevels(linebreaks), linebreaks)
}
// Return multiline reordering array for a given level array. Reordering
// does not occur across a line break.
func computeMultilineReordering(levels []level, linebreaks []int) []int {
result := make([]int, len(levels))
start := 0
for _, limit := range linebreaks {
tempLevels := make([]level, limit-start)
copy(tempLevels, levels[start:])
for j, order := range computeReordering(tempLevels) {
result[start+j] = order + start
}
start = limit
}
return result
}
// Return reordering array for a given level array. This reorders a single
// line. The reordering is a visual to logical map. For example, the
// leftmost char is string.charAt(order[0]). Rule L2.
func computeReordering(levels []level) []int {
result := make([]int, len(levels))
// initialize order
for i := range result {
result[i] = i
}
// locate highest level found on line.
// Note the rules say text, but no reordering across line bounds is
// performed, so this is sufficient.
highestLevel := level(0)
lowestOddLevel := level(maxDepth + 2)
for _, level := range levels {
if level > highestLevel {
highestLevel = level
}
if level&1 != 0 && level < lowestOddLevel {
lowestOddLevel = level
}
}
for level := highestLevel; level >= lowestOddLevel; level-- {
for i := 0; i < len(levels); i++ {
if levels[i] >= level {
// find range of text at or above this level
start := i
limit := i + 1
for limit < len(levels) && levels[limit] >= level {
limit++
}
for j, k := start, limit-1; j < k; j, k = j+1, k-1 {
result[j], result[k] = result[k], result[j]
}
// skip to end of level run
i = limit
}
}
}
return result
}
// isWhitespace reports whether the type is considered a whitespace type for the
// line break rules.
func isWhitespace(c Class) bool {
switch c {
case LRE, RLE, LRO, RLO, PDF, LRI, RLI, FSI, PDI, BN, WS:
return true
}
return false
}
// isRemovedByX9 reports whether the type is one of the types removed in X9.
func isRemovedByX9(c Class) bool {
switch c {
case LRE, RLE, LRO, RLO, PDF, BN:
return true
}
return false
}
// typeForLevel reports the strong type (L or R) corresponding to the level.
func typeForLevel(level level) Class {
if (level & 0x1) == 0 {
return L
}
return R
}
// TODO: change validation to not panic
func validateTypes(types []Class) {
if len(types) == 0 {
log.Panic("types is null")
}
for i, t := range types[:len(types)-1] {
if t == B {
log.Panicf("B type before end of paragraph at index: %d", i)
}
}
}
func validateParagraphEmbeddingLevel(embeddingLevel level) {
if embeddingLevel != implicitLevel &&
embeddingLevel != 0 &&
embeddingLevel != 1 {
log.Panicf("illegal paragraph embedding level: %d", embeddingLevel)
}
}
func validateLineBreaks(linebreaks []int, textLength int) {
prev := 0
for i, next := range linebreaks {
if next <= prev {
log.Panicf("bad linebreak: %d at index: %d", next, i)
}
prev = next
}
if prev != textLength {
log.Panicf("last linebreak was %d, want %d", prev, textLength)
}
}
func validatePbTypes(pairTypes []bracketType) {
if len(pairTypes) == 0 {
log.Panic("pairTypes is null")
}
for i, pt := range pairTypes {
switch pt {
case bpNone, bpOpen, bpClose:
default:
log.Panicf("illegal pairType value at %d: %v", i, pairTypes[i])
}
}
}
func validatePbValues(pairValues []rune, pairTypes []bracketType) {
if pairValues == nil {
log.Panic("pairValues is null")
}
if len(pairTypes) != len(pairValues) {
log.Panic("pairTypes is different length from pairValues")
}
}
vendor/golang.org/x/text/unicode/bidi/prop.go 0 → 100644
View file @ 5bee5d69
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bidi
import "unicode/utf8"
// Properties provides access to BiDi properties of runes.
type Properties struct {
entry uint8
last uint8
}
var trie = newBidiTrie(0)
// TODO: using this for bidirule reduces the running time by about 5%. Consider
// if this is worth exposing or if we can find a way to speed up the Class
// method.
//
// // CompactClass is like Class, but maps all of the BiDi control classes
// // (LRO, RLO, LRE, RLE, PDF, LRI, RLI, FSI, PDI) to the class Control.
// func (p Properties) CompactClass() Class {
// return Class(p.entry & 0x0F)
// }
// Class returns the Bidi class for p.
func (p Properties) Class() Class {
c := Class(p.entry & 0x0F)
if c == Control {
c = controlByteToClass[p.last&0xF]
}
return c
}
// IsBracket reports whether the rune is a bracket.
func (p Properties) IsBracket() bool { return p.entry&0xF0 != 0 }
// IsOpeningBracket reports whether the rune is an opening bracket.
// IsBracket must return true.
func (p Properties) IsOpeningBracket() bool { return p.entry&openMask != 0 }
// TODO: find a better API and expose.
func (p Properties) reverseBracket(r rune) rune {
return xorMasks[p.entry>>xorMaskShift] ^ r
}
var controlByteToClass = [16]Class{
0xD: LRO, // U+202D LeftToRightOverride,
0xE: RLO, // U+202E RightToLeftOverride,
0xA: LRE, // U+202A LeftToRightEmbedding,
0xB: RLE, // U+202B RightToLeftEmbedding,
0xC: PDF, // U+202C PopDirectionalFormat,
0x6: LRI, // U+2066 LeftToRightIsolate,
0x7: RLI, // U+2067 RightToLeftIsolate,
0x8: FSI, // U+2068 FirstStrongIsolate,
0x9: PDI, // U+2069 PopDirectionalIsolate,
}
// LookupRune returns properties for r.
func LookupRune(r rune) (p Properties, size int) {
var buf [4]byte
n := utf8.EncodeRune(buf[:], r)
return Lookup(buf[:n])
}
// TODO: these lookup methods are based on the generated trie code. The returned
// sizes have slightly different semantics from the generated code, in that it
// always returns size==1 for an illegal UTF-8 byte (instead of the length
// of the maximum invalid subsequence). Most Transformers, like unicode/norm,
// leave invalid UTF-8 untouched, in which case it has performance benefits to
// do so (without changing the semantics). Bidi requires the semantics used here
// for the bidirule implementation to be compatible with the Go semantics.
// They ultimately should perhaps be adopted by all trie implementations, for
// convenience sake.
// This unrolled code also boosts performance of the secure/bidirule package by
// about 30%.
// So, to remove this code:
// - add option to trie generator to define return type.
// - always return 1 byte size for ill-formed UTF-8 runes.
// Lookup returns properties for the first rune in s and the width in bytes of
// its encoding. The size will be 0 if s does not hold enough bytes to complete
// the encoding.
func Lookup(s []byte) (p Properties, sz int) {
c0 := s[0]
switch {
case c0 < 0x80: // is ASCII
return Properties{entry: bidiValues[c0]}, 1
case c0 < 0xC2:
return Properties{}, 1
case c0 < 0xE0: // 2-byte UTF-8
if len(s) < 2 {
return Properties{}, 0
}
i := bidiIndex[c0]
c1 := s[1]
if c1 < 0x80 || 0xC0 <= c1 {
return Properties{}, 1
}
return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
case c0 < 0xF0: // 3-byte UTF-8
if len(s) < 3 {
return Properties{}, 0
}
i := bidiIndex[c0]
c1 := s[1]
if c1 < 0x80 || 0xC0 <= c1 {
return Properties{}, 1
}
o := uint32(i)<<6 + uint32(c1)
i = bidiIndex[o]
c2 := s[2]
if c2 < 0x80 || 0xC0 <= c2 {
return Properties{}, 1
}
return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
case c0 < 0xF8: // 4-byte UTF-8
if len(s) < 4 {
return Properties{}, 0
}
i := bidiIndex[c0]
c1 := s[1]
if c1 < 0x80 || 0xC0 <= c1 {
return Properties{}, 1
}
o := uint32(i)<<6 + uint32(c1)
i = bidiIndex[o]
c2 := s[2]
if c2 < 0x80 || 0xC0 <= c2 {
return Properties{}, 1
}
o = uint32(i)<<6 + uint32(c2)
i = bidiIndex[o]
c3 := s[3]
if c3 < 0x80 || 0xC0 <= c3 {
return Properties{}, 1
}
return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
}
// Illegal rune
return Properties{}, 1
}
// LookupString returns properties for the first rune in s and the width in
// bytes of its encoding. The size will be 0 if s does not hold enough bytes to
// complete the encoding.
func LookupString(s string) (p Properties, sz int) {
c0 := s[0]
switch {
case c0 < 0x80: // is ASCII
return Properties{entry: bidiValues[c0]}, 1
case c0 < 0xC2:
return Properties{}, 1
case c0 < 0xE0: // 2-byte UTF-8
if len(s) < 2 {
return Properties{}, 0
}
i := bidiIndex[c0]
c1 := s[1]
if c1 < 0x80 || 0xC0 <= c1 {
return Properties{}, 1
}
return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
case c0 < 0xF0: // 3-byte UTF-8
if len(s) < 3 {
return Properties{}, 0
}
i := bidiIndex[c0]
c1 := s[1]
if c1 < 0x80 || 0xC0 <= c1 {
return Properties{}, 1
}
o := uint32(i)<<6 + uint32(c1)
i = bidiIndex[o]
c2 := s[2]
if c2 < 0x80 || 0xC0 <= c2 {
return Properties{}, 1
}
return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
case c0 < 0xF8: // 4-byte UTF-8
if len(s) < 4 {
return Properties{}, 0
}
i := bidiIndex[c0]
c1 := s[1]
if c1 < 0x80 || 0xC0 <= c1 {
return Properties{}, 1
}
o := uint32(i)<<6 + uint32(c1)
i = bidiIndex[o]
c2 := s[2]
if c2 < 0x80 || 0xC0 <= c2 {
return Properties{}, 1
}
o = uint32(i)<<6 + uint32(c2)
i = bidiIndex[o]
c3 := s[3]
if c3 < 0x80 || 0xC0 <= c3 {
return Properties{}, 1
}
return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
}
// Illegal rune
return Properties{}, 1
}
vendor/golang.org/x/text/unicode/bidi/tables10.0.0.go 0 → 100644
View file @ 5bee5d69
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/golang.org/x/text/unicode/bidi/tables9.0.0.go 0 → 100644
View file @ 5bee5d69
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/golang.org/x/text/unicode/bidi/trieval.go 0 → 100644
View file @ 5bee5d69
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
package bidi
// Class is the Unicode BiDi class. Each rune has a single class.
type Class uint
const (
L Class = iota // LeftToRight
R // RightToLeft
EN // EuropeanNumber
ES // EuropeanSeparator
ET // EuropeanTerminator
AN // ArabicNumber
CS // CommonSeparator
B // ParagraphSeparator
S // SegmentSeparator
WS // WhiteSpace
ON // OtherNeutral
BN // BoundaryNeutral
NSM // NonspacingMark
AL // ArabicLetter
Control // Control LRO - PDI
numClass
LRO // LeftToRightOverride
RLO // RightToLeftOverride
LRE // LeftToRightEmbedding
RLE // RightToLeftEmbedding
PDF // PopDirectionalFormat
LRI // LeftToRightIsolate
RLI // RightToLeftIsolate
FSI // FirstStrongIsolate
PDI // PopDirectionalIsolate
unknownClass = ^Class(0)
)
var controlToClass = map[rune]Class{
0x202D: LRO, // LeftToRightOverride,
0x202E: RLO, // RightToLeftOverride,
0x202A: LRE, // LeftToRightEmbedding,
0x202B: RLE, // RightToLeftEmbedding,
0x202C: PDF, // PopDirectionalFormat,
0x2066: LRI, // LeftToRightIsolate,
0x2067: RLI, // RightToLeftIsolate,
0x2068: FSI, // FirstStrongIsolate,
0x2069: PDI, // PopDirectionalIsolate,
}
// A trie entry has the following bits:
// 7..5 XOR mask for brackets
// 4 1: Bracket open, 0: Bracket close
// 3..0 Class type
const (
openMask = 0x10
xorMaskShift = 5
)
vendor/golang.org/x/text/unicode/norm/composition.go 0 → 100644
View file @ 5bee5d69
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package norm
import "unicode/utf8"
const (
maxNonStarters = 30
// The maximum number of characters needed for a buffer is
// maxNonStarters + 1 for the starter + 1 for the GCJ
maxBufferSize = maxNonStarters + 2
maxNFCExpansion = 3 // NFC(0x1D160)
maxNFKCExpansion = 18 // NFKC(0xFDFA)
maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128
)
// ssState is used for reporting the segment state after inserting a rune.
// It is returned by streamSafe.next.
type ssState int
const (
// Indicates a rune was successfully added to the segment.
ssSuccess ssState = iota
// Indicates a rune starts a new segment and should not be added.
ssStarter
// Indicates a rune caused a segment overflow and a CGJ should be inserted.
ssOverflow
)
// streamSafe implements the policy of when a CGJ should be inserted.
type streamSafe uint8
// first inserts the first rune of a segment. It is a faster version of next if
// it is known p represents the first rune in a segment.
func (ss *streamSafe) first(p Properties) {
*ss = streamSafe(p.nTrailingNonStarters())
}
// insert returns a ssState value to indicate whether a rune represented by p
// can be inserted.
func (ss *streamSafe) next(p Properties) ssState {
if *ss > maxNonStarters {
panic("streamSafe was not reset")
}
n := p.nLeadingNonStarters()
if *ss += streamSafe(n); *ss > maxNonStarters {
*ss = 0
return ssOverflow
}
// The Stream-Safe Text Processing prescribes that the counting can stop
// as soon as a starter is encountered. However, there are some starters,
// like Jamo V and T, that can combine with other runes, leaving their
// successive non-starters appended to the previous, possibly causing an
// overflow. We will therefore consider any rune with a non-zero nLead to
// be a non-starter. Note that it always hold that if nLead > 0 then
// nLead == nTrail.
if n == 0 {
*ss = streamSafe(p.nTrailingNonStarters())
return ssStarter
}
return ssSuccess
}
// backwards is used for checking for overflow and segment starts
// when traversing a string backwards. Users do not need to call first
// for the first rune. The state of the streamSafe retains the count of
// the non-starters loaded.
func (ss *streamSafe) backwards(p Properties) ssState {
if *ss > maxNonStarters {
panic("streamSafe was not reset")
}
c := *ss + streamSafe(p.nTrailingNonStarters())
if c > maxNonStarters {
return ssOverflow
}
*ss = c
if p.nLeadingNonStarters() == 0 {
return ssStarter
}
return ssSuccess
}
func (ss streamSafe) isMax() bool {
return ss == maxNonStarters
}
// GraphemeJoiner is inserted after maxNonStarters non-starter runes.
const GraphemeJoiner = "\u034F"
// reorderBuffer is used to normalize a single segment. Characters inserted with
// insert are decomposed and reordered based on CCC. The compose method can
// be used to recombine characters. Note that the byte buffer does not hold
// the UTF-8 characters in order. Only the rune array is maintained in sorted
// order. flush writes the resulting segment to a byte array.
type reorderBuffer struct {
rune [maxBufferSize]Properties // Per character info.
byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos.
nbyte uint8 // Number or bytes.
ss streamSafe // For limiting length of non-starter sequence.
nrune int // Number of runeInfos.
f formInfo
src input
nsrc int
tmpBytes input
out []byte
flushF func(*reorderBuffer) bool
}
func (rb *reorderBuffer) init(f Form, src []byte) {
rb.f = *formTable[f]
rb.src.setBytes(src)
rb.nsrc = len(src)
rb.ss = 0
}
func (rb *reorderBuffer) initString(f Form, src string) {
rb.f = *formTable[f]
rb.src.setString(src)
rb.nsrc = len(src)
rb.ss = 0
}
func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) {
rb.out = out
rb.flushF = f
}
// reset discards all characters from the buffer.
func (rb *reorderBuffer) reset() {
rb.nrune = 0
rb.nbyte = 0
}
func (rb *reorderBuffer) doFlush() bool {
if rb.f.composing {
rb.compose()
}
res := rb.flushF(rb)
rb.reset()
return res
}
// appendFlush appends the normalized segment to rb.out.
func appendFlush(rb *reorderBuffer) bool {
for i := 0; i < rb.nrune; i++ {
start := rb.rune[i].pos
end := start + rb.rune[i].size
rb.out = append(rb.out, rb.byte[start:end]...)
}
return true
}
// flush appends the normalized segment to out and resets rb.
func (rb *reorderBuffer) flush(out []byte) []byte {
for i := 0; i < rb.nrune; i++ {
start := rb.rune[i].pos
end := start + rb.rune[i].size
out = append(out, rb.byte[start:end]...)
}
rb.reset()
return out
}
// flushCopy copies the normalized segment to buf and resets rb.
// It returns the number of bytes written to buf.
func (rb *reorderBuffer) flushCopy(buf []byte) int {
p := 0
for i := 0; i < rb.nrune; i++ {
runep := rb.rune[i]
p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size])
}
rb.reset()
return p
}
// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class.
// It returns false if the buffer is not large enough to hold the rune.
// It is used internally by insert and insertString only.
func (rb *reorderBuffer) insertOrdered(info Properties) {
n := rb.nrune
b := rb.rune[:]
cc := info.ccc
if cc > 0 {
// Find insertion position + move elements to make room.
for ; n > 0; n-- {
if b[n-1].ccc <= cc {
break
}
b[n] = b[n-1]
}
}
rb.nrune += 1
pos := uint8(rb.nbyte)
rb.nbyte += utf8.UTFMax
info.pos = pos
b[n] = info
}
// insertErr is an error code returned by insert. Using this type instead
// of error improves performance up to 20% for many of the benchmarks.
type insertErr int
const (
iSuccess insertErr = -iota
iShortDst
iShortSrc
)
// insertFlush inserts the given rune in the buffer ordered by CCC.
// If a decomposition with multiple segments are encountered, they leading
// ones are flushed.
// It returns a non-zero error code if the rune was not inserted.
func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr {
if rune := src.hangul(i); rune != 0 {
rb.decomposeHangul(rune)
return iSuccess
}
if info.hasDecomposition() {
return rb.insertDecomposed(info.Decomposition())
}
rb.insertSingle(src, i, info)
return iSuccess
}
// insertUnsafe inserts the given rune in the buffer ordered by CCC.
// It is assumed there is sufficient space to hold the runes. It is the
// responsibility of the caller to ensure this. This can be done by checking
// the state returned by the streamSafe type.
func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) {
if rune := src.hangul(i); rune != 0 {
rb.decomposeHangul(rune)
}
if info.hasDecomposition() {
// TODO: inline.
rb.insertDecomposed(info.Decomposition())
} else {
rb.insertSingle(src, i, info)
}
}
// insertDecomposed inserts an entry in to the reorderBuffer for each rune
// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes.
// It flushes the buffer on each new segment start.
func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr {
rb.tmpBytes.setBytes(dcomp)
// As the streamSafe accounting already handles the counting for modifiers,
// we don't have to call next. However, we do need to keep the accounting
// intact when flushing the buffer.
for i := 0; i < len(dcomp); {
info := rb.f.info(rb.tmpBytes, i)
if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() {
return iShortDst
}
i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)])
rb.insertOrdered(info)
}
return iSuccess
}
// insertSingle inserts an entry in the reorderBuffer for the rune at
// position i. info is the runeInfo for the rune at position i.
func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) {
src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size))
rb.insertOrdered(info)
}
// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb.
func (rb *reorderBuffer) insertCGJ() {
rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))})
}
// appendRune inserts a rune at the end of the buffer. It is used for Hangul.
func (rb *reorderBuffer) appendRune(r rune) {
bn := rb.nbyte
sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
rb.nbyte += utf8.UTFMax
rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)}
rb.nrune++
}
// assignRune sets a rune at position pos. It is used for Hangul and recomposition.
func (rb *reorderBuffer) assignRune(pos int, r rune) {
bn := rb.rune[pos].pos
sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
rb.rune[pos] = Properties{pos: bn, size: uint8(sz)}
}
// runeAt returns the rune at position n. It is used for Hangul and recomposition.
func (rb *reorderBuffer) runeAt(n int) rune {
inf := rb.rune[n]
r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size])
return r
}
// bytesAt returns the UTF-8 encoding of the rune at position n.
// It is used for Hangul and recomposition.
func (rb *reorderBuffer) bytesAt(n int) []byte {
inf := rb.rune[n]
return rb.byte[inf.pos : int(inf.pos)+int(inf.size)]
}
// For Hangul we combine algorithmically, instead of using tables.
const (
hangulBase = 0xAC00 // UTF-8(hangulBase) -> EA B0 80
hangulBase0 = 0xEA
hangulBase1 = 0xB0
hangulBase2 = 0x80
hangulEnd = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4
hangulEnd0 = 0xED
hangulEnd1 = 0x9E
hangulEnd2 = 0xA4
jamoLBase = 0x1100 // UTF-8(jamoLBase) -> E1 84 00
jamoLBase0 = 0xE1
jamoLBase1 = 0x84
jamoLEnd = 0x1113
jamoVBase = 0x1161
jamoVEnd = 0x1176
jamoTBase = 0x11A7
jamoTEnd = 0x11C3
jamoTCount = 28
jamoVCount = 21
jamoVTCount = 21 * 28
jamoLVTCount = 19 * 21 * 28
)
const hangulUTF8Size = 3
func isHangul(b []byte) bool {
if len(b) < hangulUTF8Size {
return false
}
b0 := b[0]
if b0 < hangulBase0 {
return false
}
b1 := b[1]
switch {
case b0 == hangulBase0:
return b1 >= hangulBase1
case b0 < hangulEnd0:
return true
case b0 > hangulEnd0:
return false
case b1 < hangulEnd1:
return true
}
return b1 == hangulEnd1 && b[2] < hangulEnd2
}
func isHangulString(b string) bool {
if len(b) < hangulUTF8Size {
return false
}
b0 := b[0]
if b0 < hangulBase0 {
return false
}
b1 := b[1]
switch {
case b0 == hangulBase0:
return b1 >= hangulBase1
case b0 < hangulEnd0:
return true
case b0 > hangulEnd0:
return false
case b1 < hangulEnd1:
return true
}
return b1 == hangulEnd1 && b[2] < hangulEnd2
}
// Caller must ensure len(b) >= 2.
func isJamoVT(b []byte) bool {
// True if (rune & 0xff00) == jamoLBase
return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1
}
func isHangulWithoutJamoT(b []byte) bool {
c, _ := utf8.DecodeRune(b)
c -= hangulBase
return c < jamoLVTCount && c%jamoTCount == 0
}
// decomposeHangul writes the decomposed Hangul to buf and returns the number
// of bytes written. len(buf) should be at least 9.
func decomposeHangul(buf []byte, r rune) int {
const JamoUTF8Len = 3
r -= hangulBase
x := r % jamoTCount
r /= jamoTCount
utf8.EncodeRune(buf, jamoLBase+r/jamoVCount)
utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount)
if x != 0 {
utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x)
return 3 * JamoUTF8Len
}
return 2 * JamoUTF8Len
}
// decomposeHangul algorithmically decomposes a Hangul rune into
// its Jamo components.
// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul.
func (rb *reorderBuffer) decomposeHangul(r rune) {
r -= hangulBase
x := r % jamoTCount
r /= jamoTCount
rb.appendRune(jamoLBase + r/jamoVCount)
rb.appendRune(jamoVBase + r%jamoVCount)
if x != 0 {
rb.appendRune(jamoTBase + x)
}
}
// combineHangul algorithmically combines Jamo character components into Hangul.
// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul.
func (rb *reorderBuffer) combineHangul(s, i, k int) {
b := rb.rune[:]
bn := rb.nrune
for ; i < bn; i++ {
cccB := b[k-1].ccc
cccC := b[i].ccc
if cccB == 0 {
s = k - 1
}
if s != k-1 && cccB >= cccC {
// b[i] is blocked by greater-equal cccX below it
b[k] = b[i]
k++
} else {
l := rb.runeAt(s) // also used to compare to hangulBase
v := rb.runeAt(i) // also used to compare to jamoT
switch {
case jamoLBase <= l && l < jamoLEnd &&
jamoVBase <= v && v < jamoVEnd:
// 11xx plus 116x to LV
rb.assignRune(s, hangulBase+
(l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount)
case hangulBase <= l && l < hangulEnd &&
jamoTBase < v && v < jamoTEnd &&
((l-hangulBase)%jamoTCount) == 0:
// ACxx plus 11Ax to LVT
rb.assignRune(s, l+v-jamoTBase)
default:
b[k] = b[i]
k++
}
}
}
rb.nrune = k
}
// compose recombines the runes in the buffer.
// It should only be used to recompose a single segment, as it will not
// handle alternations between Hangul and non-Hangul characters correctly.
func (rb *reorderBuffer) compose() {
// UAX #15, section X5 , including Corrigendum #5
// "In any character sequence beginning with starter S, a character C is
// blocked from S if and only if there is some character B between S
// and C, and either B is a starter or it has the same or higher
// combining class as C."
bn := rb.nrune
if bn == 0 {
return
}
k := 1
b := rb.rune[:]
for s, i := 0, 1; i < bn; i++ {
if isJamoVT(rb.bytesAt(i)) {
// Redo from start in Hangul mode. Necessary to support
// U+320E..U+321E in NFKC mode.
rb.combineHangul(s, i, k)
return
}
ii := b[i]
// We can only use combineForward as a filter if we later
// get the info for the combined character. This is more
// expensive than using the filter. Using combinesBackward()
// is safe.
if ii.combinesBackward() {
cccB := b[k-1].ccc
cccC := ii.ccc
blocked := false // b[i] blocked by starter or greater or equal CCC?
if cccB == 0 {
s = k - 1
} else {
blocked = s != k-1 && cccB >= cccC
}
if !blocked {
combined := combine(rb.runeAt(s), rb.runeAt(i))
if combined != 0 {
rb.assignRune(s, combined)
continue
}
}
}
b[k] = b[i]
k++
}
rb.nrune = k
}
vendor/golang.org/x/text/unicode/norm/forminfo.go 0 → 100644
View file @ 5bee5d69
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package norm
// This file contains Form-specific logic and wrappers for data in tables.go.
// Rune info is stored in a separate trie per composing form. A composing form
// and its corresponding decomposing form share the same trie. Each trie maps
// a rune to a uint16. The values take two forms. For v >= 0x8000:
// bits
// 15: 1 (inverse of NFD_QC bit of qcInfo)
// 13..7: qcInfo (see below). isYesD is always true (no decompostion).
// 6..0: ccc (compressed CCC value).
// For v < 0x8000, the respective rune has a decomposition and v is an index
// into a byte array of UTF-8 decomposition sequences and additional info and
// has the form:
// <header> <decomp_byte>* [<tccc> [<lccc>]]
// The header contains the number of bytes in the decomposition (excluding this
// length byte). The two most significant bits of this length byte correspond
// to bit 5 and 4 of qcInfo (see below). The byte sequence itself starts at v+1.
// The byte sequence is followed by a trailing and leading CCC if the values
// for these are not zero. The value of v determines which ccc are appended
// to the sequences. For v < firstCCC, there are none, for v >= firstCCC,
// the sequence is followed by a trailing ccc, and for v >= firstLeadingCC
// there is an additional leading ccc. The value of tccc itself is the
// trailing CCC shifted left 2 bits. The two least-significant bits of tccc
// are the number of trailing non-starters.
const (
qcInfoMask = 0x3F // to clear all but the relevant bits in a qcInfo
headerLenMask = 0x3F // extract the length value from the header byte
headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
)
// Properties provides access to normalization properties of a rune.
type Properties struct {
pos uint8 // start position in reorderBuffer; used in composition.go
size uint8 // length of UTF-8 encoding of this rune
ccc uint8 // leading canonical combining class (ccc if not decomposition)
tccc uint8 // trailing canonical combining class (ccc if not decomposition)
nLead uint8 // number of leading non-starters.
flags qcInfo // quick check flags
index uint16
}
// functions dispatchable per form
type lookupFunc func(b input, i int) Properties
// formInfo holds Form-specific functions and tables.
type formInfo struct {
form Form
composing, compatibility bool // form type
info lookupFunc
nextMain iterFunc
}
var formTable = []*formInfo{{
form: NFC,
composing: true,
compatibility: false,
info: lookupInfoNFC,
nextMain: nextComposed,
}, {
form: NFD,
composing: false,
compatibility: false,
info: lookupInfoNFC,
nextMain: nextDecomposed,
}, {
form: NFKC,
composing: true,
compatibility: true,
info: lookupInfoNFKC,
nextMain: nextComposed,
}, {
form: NFKD,
composing: false,
compatibility: true,
info: lookupInfoNFKC,
nextMain: nextDecomposed,
}}
// We do not distinguish between boundaries for NFC, NFD, etc. to avoid
// unexpected behavior for the user. For example, in NFD, there is a boundary
// after 'a'. However, 'a' might combine with modifiers, so from the application's
// perspective it is not a good boundary. We will therefore always use the
// boundaries for the combining variants.
// BoundaryBefore returns true if this rune starts a new segment and
// cannot combine with any rune on the left.
func (p Properties) BoundaryBefore() bool {
if p.ccc == 0 && !p.combinesBackward() {
return true
}
// We assume that the CCC of the first character in a decomposition
// is always non-zero if different from info.ccc and that we can return
// false at this point. This is verified by maketables.
return false
}
// BoundaryAfter returns true if runes cannot combine with or otherwise
// interact with this or previous runes.
func (p Properties) BoundaryAfter() bool {
// TODO: loosen these conditions.
return p.isInert()
}
// We pack quick check data in 4 bits:
// 5: Combines forward (0 == false, 1 == true)
// 4..3: NFC_QC Yes(00), No (10), or Maybe (11)
// 2: NFD_QC Yes (0) or No (1). No also means there is a decomposition.
// 1..0: Number of trailing non-starters.
//
// When all 4 bits are zero, the character is inert, meaning it is never
// influenced by normalization.
type qcInfo uint8
func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
func (p Properties) combinesForward() bool { return p.flags&0x20 != 0 }
func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe
func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD
func (p Properties) isInert() bool {
return p.flags&qcInfoMask == 0 && p.ccc == 0
}
func (p Properties) multiSegment() bool {
return p.index >= firstMulti && p.index < endMulti
}
func (p Properties) nLeadingNonStarters() uint8 {
return p.nLead
}
func (p Properties) nTrailingNonStarters() uint8 {
return uint8(p.flags & 0x03)
}
// Decomposition returns the decomposition for the underlying rune
// or nil if there is none.
func (p Properties) Decomposition() []byte {
// TODO: create the decomposition for Hangul?
if p.index == 0 {
return nil
}
i := p.index
n := decomps[i] & headerLenMask
i++
return decomps[i : i+uint16(n)]
}
// Size returns the length of UTF-8 encoding of the rune.
func (p Properties) Size() int {
return int(p.size)
}
// CCC returns the canonical combining class of the underlying rune.
func (p Properties) CCC() uint8 {
if p.index >= firstCCCZeroExcept {
return 0
}
return ccc[p.ccc]
}
// LeadCCC returns the CCC of the first rune in the decomposition.
// If there is no decomposition, LeadCCC equals CCC.
func (p Properties) LeadCCC() uint8 {
return ccc[p.ccc]
}
// TrailCCC returns the CCC of the last rune in the decomposition.
// If there is no decomposition, TrailCCC equals CCC.
func (p Properties) TrailCCC() uint8 {
return ccc[p.tccc]
}
// Recomposition
// We use 32-bit keys instead of 64-bit for the two codepoint keys.
// This clips off the bits of three entries, but we know this will not
// result in a collision. In the unlikely event that changes to
// UnicodeData.txt introduce collisions, the compiler will catch it.
// Note that the recomposition map for NFC and NFKC are identical.
// combine returns the combined rune or 0 if it doesn't exist.
func combine(a, b rune) rune {
key := uint32(uint16(a))<<16 + uint32(uint16(b))
return recompMap[key]
}
func lookupInfoNFC(b input, i int) Properties {
v, sz := b.charinfoNFC(i)
return compInfo(v, sz)
}
func lookupInfoNFKC(b input, i int) Properties {
v, sz := b.charinfoNFKC(i)
return compInfo(v, sz)
}
// Properties returns properties for the first rune in s.
func (f Form) Properties(s []byte) Properties {
if f == NFC || f == NFD {
return compInfo(nfcData.lookup(s))
}
return compInfo(nfkcData.lookup(s))
}
// PropertiesString returns properties for the first rune in s.
func (f Form) PropertiesString(s string) Properties {
if f == NFC || f == NFD {
return compInfo(nfcData.lookupString(s))
}
return compInfo(nfkcData.lookupString(s))
}
// compInfo converts the information contained in v and sz
// to a Properties. See the comment at the top of the file
// for more information on the format.
func compInfo(v uint16, sz int) Properties {
if v == 0 {
return Properties{size: uint8(sz)}
} else if v >= 0x8000 {
p := Properties{
size: uint8(sz),
ccc: uint8(v),
tccc: uint8(v),
flags: qcInfo(v >> 8),
}
if p.ccc > 0 || p.combinesBackward() {
p.nLead = uint8(p.flags & 0x3)
}
return p
}
// has decomposition
h := decomps[v]
f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
p := Properties{size: uint8(sz), flags: f, index: v}
if v >= firstCCC {
v += uint16(h&headerLenMask) + 1
c := decomps[v]
p.tccc = c >> 2
p.flags |= qcInfo(c & 0x3)
if v >= firstLeadingCCC {
p.nLead = c & 0x3
if v >= firstStarterWithNLead {
// We were tricked. Remove the decomposition.
p.flags &= 0x03
p.index = 0
return p
}
p.ccc = decomps[v+1]
}
}
return p
}
vendor/golang.org/x/text/unicode/norm/input.go 0 → 100644
View file @ 5bee5d69
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package norm
import "unicode/utf8"
type input struct {
str string
bytes []byte
}
func inputBytes(str []byte) input {
return input{bytes: str}
}
func inputString(str string) input {
return input{str: str}
}
func (in *input) setBytes(str []byte) {
in.str = ""
in.bytes = str
}
func (in *input) setString(str string) {
in.str = str
in.bytes = nil
}
func (in *input) _byte(p int) byte {
if in.bytes == nil {
return in.str[p]
}
return in.bytes[p]
}
func (in *input) skipASCII(p, max int) int {
if in.bytes == nil {
for ; p < max && in.str[p] < utf8.RuneSelf; p++ {
}
} else {
for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ {
}
}
return p
}
func (in *input) skipContinuationBytes(p int) int {
if in.bytes == nil {
for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ {
}
} else {
for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ {
}
}
return p
}
func (in *input) appendSlice(buf []byte, b, e int) []byte {
if in.bytes != nil {
return append(buf, in.bytes[b:e]...)
}
for i := b; i < e; i++ {
buf = append(buf, in.str[i])
}
return buf
}
func (in *input) copySlice(buf []byte, b, e int) int {
if in.bytes == nil {
return copy(buf, in.str[b:e])
}
return copy(buf, in.bytes[b:e])
}
func (in *input) charinfoNFC(p int) (uint16, int) {
if in.bytes == nil {
return nfcData.lookupString(in.str[p:])
}
return nfcData.lookup(in.bytes[p:])
}
func (in *input) charinfoNFKC(p int) (uint16, int) {
if in.bytes == nil {
return nfkcData.lookupString(in.str[p:])
}
return nfkcData.lookup(in.bytes[p:])
}
func (in *input) hangul(p int) (r rune) {
var size int
if in.bytes == nil {
if !isHangulString(in.str[p:]) {
return 0
}
r, size = utf8.DecodeRuneInString(in.str[p:])
} else {
if !isHangul(in.bytes[p:]) {
return 0
}
r, size = utf8.DecodeRune(in.bytes[p:])
}
if size != hangulUTF8Size {
return 0
}
return r
}
vendor/golang.org/x/text/unicode/norm/iter.go 0 → 100644
View file @ 5bee5d69
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package norm
import (
"fmt"
"unicode/utf8"
)
// MaxSegmentSize is the maximum size of a byte buffer needed to consider any
// sequence of starter and non-starter runes for the purpose of normalization.
const MaxSegmentSize = maxByteBufferSize
// An Iter iterates over a string or byte slice, while normalizing it
// to a given Form.
type Iter struct {
rb reorderBuffer
buf [maxByteBufferSize]byte
info Properties // first character saved from previous iteration
next iterFunc // implementation of next depends on form
asciiF iterFunc
p int // current position in input source
multiSeg []byte // remainder of multi-segment decomposition
}
type iterFunc func(*Iter) []byte
// Init initializes i to iterate over src after normalizing it to Form f.
func (i *Iter) Init(f Form, src []byte) {
i.p = 0
if len(src) == 0 {
i.setDone()
i.rb.nsrc = 0
return
}
i.multiSeg = nil
i.rb.init(f, src)
i.next = i.rb.f.nextMain
i.asciiF = nextASCIIBytes
i.info = i.rb.f.info(i.rb.src, i.p)
i.rb.ss.first(i.info)
}
// InitString initializes i to iterate over src after normalizing it to Form f.
func (i *Iter) InitString(f Form, src string) {
i.p = 0
if len(src) == 0 {
i.setDone()
i.rb.nsrc = 0
return
}
i.multiSeg = nil
i.rb.initString(f, src)
i.next = i.rb.f.nextMain
i.asciiF = nextASCIIString
i.info = i.rb.f.info(i.rb.src, i.p)
i.rb.ss.first(i.info)
}
// Seek sets the segment to be returned by the next call to Next to start
// at position p. It is the responsibility of the caller to set p to the
// start of a segment.
func (i *Iter) Seek(offset int64, whence int) (int64, error) {
var abs int64
switch whence {
case 0:
abs = offset
case 1:
abs = int64(i.p) + offset
case 2:
abs = int64(i.rb.nsrc) + offset
default:
return 0, fmt.Errorf("norm: invalid whence")
}
if abs < 0 {
return 0, fmt.Errorf("norm: negative position")
}
if int(abs) >= i.rb.nsrc {
i.setDone()
return int64(i.p), nil
}
i.p = int(abs)
i.multiSeg = nil
i.next = i.rb.f.nextMain
i.info = i.rb.f.info(i.rb.src, i.p)
i.rb.ss.first(i.info)
return abs, nil
}
// returnSlice returns a slice of the underlying input type as a byte slice.
// If the underlying is of type []byte, it will simply return a slice.
// If the underlying is of type string, it will copy the slice to the buffer
// and return that.
func (i *Iter) returnSlice(a, b int) []byte {
if i.rb.src.bytes == nil {
return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])]
}
return i.rb.src.bytes[a:b]
}
// Pos returns the byte position at which the next call to Next will commence processing.
func (i *Iter) Pos() int {
return i.p
}
func (i *Iter) setDone() {
i.next = nextDone
i.p = i.rb.nsrc
}
// Done returns true if there is no more input to process.
func (i *Iter) Done() bool {
return i.p >= i.rb.nsrc
}
// Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input.
// For any input a and b for which f(a) == f(b), subsequent calls
// to Next will return the same segments.
// Modifying runes are grouped together with the preceding starter, if such a starter exists.
// Although not guaranteed, n will typically be the smallest possible n.
func (i *Iter) Next() []byte {
return i.next(i)
}
func nextASCIIBytes(i *Iter) []byte {
p := i.p + 1
if p >= i.rb.nsrc {
i.setDone()
return i.rb.src.bytes[i.p:p]
}
if i.rb.src.bytes[p] < utf8.RuneSelf {
p0 := i.p
i.p = p
return i.rb.src.bytes[p0:p]
}
i.info = i.rb.f.info(i.rb.src, i.p)
i.next = i.rb.f.nextMain
return i.next(i)
}
func nextASCIIString(i *Iter) []byte {
p := i.p + 1
if p >= i.rb.nsrc {
i.buf[0] = i.rb.src.str[i.p]
i.setDone()
return i.buf[:1]
}
if i.rb.src.str[p] < utf8.RuneSelf {
i.buf[0] = i.rb.src.str[i.p]
i.p = p
return i.buf[:1]
}
i.info = i.rb.f.info(i.rb.src, i.p)
i.next = i.rb.f.nextMain
return i.next(i)
}
func nextHangul(i *Iter) []byte {
p := i.p
next := p + hangulUTF8Size
if next >= i.rb.nsrc {
i.setDone()
} else if i.rb.src.hangul(next) == 0 {
i.rb.ss.next(i.info)
i.info = i.rb.f.info(i.rb.src, i.p)
i.next = i.rb.f.nextMain
return i.next(i)
}
i.p = next
return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))]
}
func nextDone(i *Iter) []byte {
return nil
}
// nextMulti is used for iterating over multi-segment decompositions
// for decomposing normal forms.
func nextMulti(i *Iter) []byte {
j := 0
d := i.multiSeg
// skip first rune
for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ {
}
for j < len(d) {
info := i.rb.f.info(input{bytes: d}, j)
if info.BoundaryBefore() {
i.multiSeg = d[j:]
return d[:j]
}
j += int(info.size)
}
// treat last segment as normal decomposition
i.next = i.rb.f.nextMain
return i.next(i)
}
// nextMultiNorm is used for iterating over multi-segment decompositions
// for composing normal forms.
func nextMultiNorm(i *Iter) []byte {
j := 0
d := i.multiSeg
for j < len(d) {
info := i.rb.f.info(input{bytes: d}, j)
if info.BoundaryBefore() {
i.rb.compose()
seg := i.buf[:i.rb.flushCopy(i.buf[:])]
i.rb.insertUnsafe(input{bytes: d}, j, info)
i.multiSeg = d[j+int(info.size):]
return seg
}
i.rb.insertUnsafe(input{bytes: d}, j, info)
j += int(info.size)
}
i.multiSeg = nil
i.next = nextComposed
return doNormComposed(i)
}
// nextDecomposed is the implementation of Next for forms NFD and NFKD.
func nextDecomposed(i *Iter) (next []byte) {
outp := 0
inCopyStart, outCopyStart := i.p, 0
for {
if sz := int(i.info.size); sz <= 1 {
i.rb.ss = 0
p := i.p
i.p++ // ASCII or illegal byte. Either way, advance by 1.
if i.p >= i.rb.nsrc {
i.setDone()
return i.returnSlice(p, i.p)
} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
i.next = i.asciiF
return i.returnSlice(p, i.p)
}
outp++
} else if d := i.info.Decomposition(); d != nil {
// Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero.
// Case 1: there is a leftover to copy. In this case the decomposition
// must begin with a modifier and should always be appended.
// Case 2: no leftover. Simply return d if followed by a ccc == 0 value.
p := outp + len(d)
if outp > 0 {
i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
// TODO: this condition should not be possible, but we leave it
// in for defensive purposes.
if p > len(i.buf) {
return i.buf[:outp]
}
} else if i.info.multiSegment() {
// outp must be 0 as multi-segment decompositions always
// start a new segment.
if i.multiSeg == nil {
i.multiSeg = d
i.next = nextMulti
return nextMulti(i)
}
// We are in the last segment. Treat as normal decomposition.
d = i.multiSeg
i.multiSeg = nil
p = len(d)
}
prevCC := i.info.tccc
if i.p += sz; i.p >= i.rb.nsrc {
i.setDone()
i.info = Properties{} // Force BoundaryBefore to succeed.
} else {
i.info = i.rb.f.info(i.rb.src, i.p)
}
switch i.rb.ss.next(i.info) {
case ssOverflow:
i.next = nextCGJDecompose
fallthrough
case ssStarter:
if outp > 0 {
copy(i.buf[outp:], d)
return i.buf[:p]
}
return d
}
copy(i.buf[outp:], d)
outp = p
inCopyStart, outCopyStart = i.p, outp
if i.info.ccc < prevCC {
goto doNorm
}
continue
} else if r := i.rb.src.hangul(i.p); r != 0 {
outp = decomposeHangul(i.buf[:], r)
i.p += hangulUTF8Size
inCopyStart, outCopyStart = i.p, outp
if i.p >= i.rb.nsrc {
i.setDone()
break
} else if i.rb.src.hangul(i.p) != 0 {
i.next = nextHangul
return i.buf[:outp]
}
} else {
p := outp + sz
if p > len(i.buf) {
break
}
outp = p
i.p += sz
}
if i.p >= i.rb.nsrc {
i.setDone()
break
}
prevCC := i.info.tccc
i.info = i.rb.f.info(i.rb.src, i.p)
if v := i.rb.ss.next(i.info); v == ssStarter {
break
} else if v == ssOverflow {
i.next = nextCGJDecompose
break
}
if i.info.ccc < prevCC {
goto doNorm
}
}
if outCopyStart == 0 {
return i.returnSlice(inCopyStart, i.p)
} else if inCopyStart < i.p {
i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
}
return i.buf[:outp]
doNorm:
// Insert what we have decomposed so far in the reorderBuffer.
// As we will only reorder, there will always be enough room.
i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
i.rb.insertDecomposed(i.buf[0:outp])
return doNormDecomposed(i)
}
func doNormDecomposed(i *Iter) []byte {
for {
i.rb.insertUnsafe(i.rb.src, i.p, i.info)
if i.p += int(i.info.size); i.p >= i.rb.nsrc {
i.setDone()
break
}
i.info = i.rb.f.info(i.rb.src, i.p)
if i.info.ccc == 0 {
break
}
if s := i.rb.ss.next(i.info); s == ssOverflow {
i.next = nextCGJDecompose
break
}
}
// new segment or too many combining characters: exit normalization
return i.buf[:i.rb.flushCopy(i.buf[:])]
}
func nextCGJDecompose(i *Iter) []byte {
i.rb.ss = 0
i.rb.insertCGJ()
i.next = nextDecomposed
i.rb.ss.first(i.info)
buf := doNormDecomposed(i)
return buf
}
// nextComposed is the implementation of Next for forms NFC and NFKC.
func nextComposed(i *Iter) []byte {
outp, startp := 0, i.p
var prevCC uint8
for {
if !i.info.isYesC() {
goto doNorm
}
prevCC = i.info.tccc
sz := int(i.info.size)
if sz == 0 {
sz = 1 // illegal rune: copy byte-by-byte
}
p := outp + sz
if p > len(i.buf) {
break
}
outp = p
i.p += sz
if i.p >= i.rb.nsrc {
i.setDone()
break
} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
i.rb.ss = 0
i.next = i.asciiF
break
}
i.info = i.rb.f.info(i.rb.src, i.p)
if v := i.rb.ss.next(i.info); v == ssStarter {
break
} else if v == ssOverflow {
i.next = nextCGJCompose
break
}
if i.info.ccc < prevCC {
goto doNorm
}
}
return i.returnSlice(startp, i.p)
doNorm:
// reset to start position
i.p = startp
i.info = i.rb.f.info(i.rb.src, i.p)
i.rb.ss.first(i.info)
if i.info.multiSegment() {
d := i.info.Decomposition()
info := i.rb.f.info(input{bytes: d}, 0)
i.rb.insertUnsafe(input{bytes: d}, 0, info)
i.multiSeg = d[int(info.size):]
i.next = nextMultiNorm
return nextMultiNorm(i)
}
i.rb.ss.first(i.info)
i.rb.insertUnsafe(i.rb.src, i.p, i.info)
return doNormComposed(i)
}
func doNormComposed(i *Iter) []byte {
// First rune should already be inserted.
for {
if i.p += int(i.info.size); i.p >= i.rb.nsrc {
i.setDone()
break
}
i.info = i.rb.f.info(i.rb.src, i.p)
if s := i.rb.ss.next(i.info); s == ssStarter {
break
} else if s == ssOverflow {
i.next = nextCGJCompose
break
}
i.rb.insertUnsafe(i.rb.src, i.p, i.info)
}
i.rb.compose()
seg := i.buf[:i.rb.flushCopy(i.buf[:])]
return seg
}
func nextCGJCompose(i *Iter) []byte {
i.rb.ss = 0 // instead of first
i.rb.insertCGJ()
i.next = nextComposed
// Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter,
// even if they are not. This is particularly dubious for U+FF9E and UFF9A.
// If we ever change that, insert a check here.
i.rb.ss.first(i.info)
i.rb.insertUnsafe(i.rb.src, i.p, i.info)
return doNormComposed(i)
}
vendor/golang.org/x/text/unicode/norm/normalize.go 0 → 100644
View file @ 5bee5d69
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Note: the file data_test.go that is generated should not be checked in.
//go:generate go run maketables.go triegen.go
//go:generate go test -tags test
// Package norm contains types and functions for normalizing Unicode strings.
package norm // import "golang.org/x/text/unicode/norm"
import (
"unicode/utf8"
"golang.org/x/text/transform"
)
// A Form denotes a canonical representation of Unicode code points.
// The Unicode-defined normalization and equivalence forms are:
//
// NFC Unicode Normalization Form C
// NFD Unicode Normalization Form D
// NFKC Unicode Normalization Form KC
// NFKD Unicode Normalization Form KD
//
// For a Form f, this documentation uses the notation f(x) to mean
// the bytes or string x converted to the given form.
// A position n in x is called a boundary if conversion to the form can
// proceed independently on both sides:
// f(x) == append(f(x[0:n]), f(x[n:])...)
//
// References: http://unicode.org/reports/tr15/ and
// http://unicode.org/notes/tn5/.
type Form int
const (
NFC Form = iota
NFD
NFKC
NFKD
)
// Bytes returns f(b). May return b if f(b) = b.
func (f Form) Bytes(b []byte) []byte {
src := inputBytes(b)
ft := formTable[f]
n, ok := ft.quickSpan(src, 0, len(b), true)
if ok {
return b
}
out := make([]byte, n, len(b))
copy(out, b[0:n])
rb := reorderBuffer{f: *ft, src: src, nsrc: len(b), out: out, flushF: appendFlush}
return doAppendInner(&rb, n)
}
// String returns f(s).
func (f Form) String(s string) string {
src := inputString(s)
ft := formTable[f]
n, ok := ft.quickSpan(src, 0, len(s), true)
if ok {
return s
}
out := make([]byte, n, len(s))
copy(out, s[0:n])
rb := reorderBuffer{f: *ft, src: src, nsrc: len(s), out: out, flushF: appendFlush}
return string(doAppendInner(&rb, n))
}
// IsNormal returns true if b == f(b).
func (f Form) IsNormal(b []byte) bool {
src := inputBytes(b)
ft := formTable[f]
bp, ok := ft.quickSpan(src, 0, len(b), true)
if ok {
return true
}
rb := reorderBuffer{f: *ft, src: src, nsrc: len(b)}
rb.setFlusher(nil, cmpNormalBytes)
for bp < len(b) {
rb.out = b[bp:]
if bp = decomposeSegment(&rb, bp, true); bp < 0 {
return false
}
bp, _ = rb.f.quickSpan(rb.src, bp, len(b), true)
}
return true
}
func cmpNormalBytes(rb *reorderBuffer) bool {
b := rb.out
for i := 0; i < rb.nrune; i++ {
info := rb.rune[i]
if int(info.size) > len(b) {
return false
}
p := info.pos
pe := p + info.size
for ; p < pe; p++ {
if b[0] != rb.byte[p] {
return false
}
b = b[1:]
}
}
return true
}
// IsNormalString returns true if s == f(s).
func (f Form) IsNormalString(s string) bool {
src := inputString(s)
ft := formTable[f]
bp, ok := ft.quickSpan(src, 0, len(s), true)
if ok {
return true
}
rb := reorderBuffer{f: *ft, src: src, nsrc: len(s)}
rb.setFlusher(nil, func(rb *reorderBuffer) bool {
for i := 0; i < rb.nrune; i++ {
info := rb.rune[i]
if bp+int(info.size) > len(s) {
return false
}
p := info.pos
pe := p + info.size
for ; p < pe; p++ {
if s[bp] != rb.byte[p] {
return false
}
bp++
}
}
return true
})
for bp < len(s) {
if bp = decomposeSegment(&rb, bp, true); bp < 0 {
return false
}
bp, _ = rb.f.quickSpan(rb.src, bp, len(s), true)
}
return true
}
// patchTail fixes a case where a rune may be incorrectly normalized
// if it is followed by illegal continuation bytes. It returns the
// patched buffer and whether the decomposition is still in progress.
func patchTail(rb *reorderBuffer) bool {
info, p := lastRuneStart(&rb.f, rb.out)
if p == -1 || info.size == 0 {
return true
}
end := p + int(info.size)
extra := len(rb.out) - end
if extra > 0 {
// Potentially allocating memory. However, this only
// happens with ill-formed UTF-8.
x := make([]byte, 0)
x = append(x, rb.out[len(rb.out)-extra:]...)
rb.out = rb.out[:end]
decomposeToLastBoundary(rb)
rb.doFlush()
rb.out = append(rb.out, x...)
return false
}
buf := rb.out[p:]
rb.out = rb.out[:p]
decomposeToLastBoundary(rb)
if s := rb.ss.next(info); s == ssStarter {
rb.doFlush()
rb.ss.first(info)
} else if s == ssOverflow {
rb.doFlush()
rb.insertCGJ()
rb.ss = 0
}
rb.insertUnsafe(inputBytes(buf), 0, info)
return true
}
func appendQuick(rb *reorderBuffer, i int) int {
if rb.nsrc == i {
return i
}
end, _ := rb.f.quickSpan(rb.src, i, rb.nsrc, true)
rb.out = rb.src.appendSlice(rb.out, i, end)
return end
}
// Append returns f(append(out, b...)).
// The buffer out must be nil, empty, or equal to f(out).
func (f Form) Append(out []byte, src ...byte) []byte {
return f.doAppend(out, inputBytes(src), len(src))
}
func (f Form) doAppend(out []byte, src input, n int) []byte {
if n == 0 {
return out
}
ft := formTable[f]
// Attempt to do a quickSpan first so we can avoid initializing the reorderBuffer.
if len(out) == 0 {
p, _ := ft.quickSpan(src, 0, n, true)
out = src.appendSlice(out, 0, p)
if p == n {
return out
}
rb := reorderBuffer{f: *ft, src: src, nsrc: n, out: out, flushF: appendFlush}
return doAppendInner(&rb, p)
}
rb := reorderBuffer{f: *ft, src: src, nsrc: n}
return doAppend(&rb, out, 0)
}
func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
rb.setFlusher(out, appendFlush)
src, n := rb.src, rb.nsrc
doMerge := len(out) > 0
if q := src.skipContinuationBytes(p); q > p {
// Move leading non-starters to destination.
rb.out = src.appendSlice(rb.out, p, q)
p = q
doMerge = patchTail(rb)
}
fd := &rb.f
if doMerge {
var info Properties
if p < n {
info = fd.info(src, p)
if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 {
if p == 0 {
decomposeToLastBoundary(rb)
}
p = decomposeSegment(rb, p, true)
}
}
if info.size == 0 {
rb.doFlush()
// Append incomplete UTF-8 encoding.
return src.appendSlice(rb.out, p, n)
}
if rb.nrune > 0 {
return doAppendInner(rb, p)
}
}
p = appendQuick(rb, p)
return doAppendInner(rb, p)
}
func doAppendInner(rb *reorderBuffer, p int) []byte {
for n := rb.nsrc; p < n; {
p = decomposeSegment(rb, p, true)
p = appendQuick(rb, p)
}
return rb.out
}
// AppendString returns f(append(out, []byte(s))).
// The buffer out must be nil, empty, or equal to f(out).
func (f Form) AppendString(out []byte, src string) []byte {
return f.doAppend(out, inputString(src), len(src))
}
// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]).
// It is not guaranteed to return the largest such n.
func (f Form) QuickSpan(b []byte) int {
n, _ := formTable[f].quickSpan(inputBytes(b), 0, len(b), true)
return n
}
// Span implements transform.SpanningTransformer. It returns a boundary n such
// that b[0:n] == f(b[0:n]). It is not guaranteed to return the largest such n.
func (f Form) Span(b []byte, atEOF bool) (n int, err error) {
n, ok := formTable[f].quickSpan(inputBytes(b), 0, len(b), atEOF)
if n < len(b) {
if !ok {
err = transform.ErrEndOfSpan
} else {
err = transform.ErrShortSrc
}
}
return n, err
}
// SpanString returns a boundary n such that s[0:n] == f(s[0:n]).
// It is not guaranteed to return the largest such n.
func (f Form) SpanString(s string, atEOF bool) (n int, err error) {
n, ok := formTable[f].quickSpan(inputString(s), 0, len(s), atEOF)
if n < len(s) {
if !ok {
err = transform.ErrEndOfSpan
} else {
err = transform.ErrShortSrc
}
}
return n, err
}
// quickSpan returns a boundary n such that src[0:n] == f(src[0:n]) and
// whether any non-normalized parts were found. If atEOF is false, n will
// not point past the last segment if this segment might be become
// non-normalized by appending other runes.
func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool) {
var lastCC uint8
ss := streamSafe(0)
lastSegStart := i
for n = end; i < n; {
if j := src.skipASCII(i, n); i != j {
i = j
lastSegStart = i - 1
lastCC = 0
ss = 0
continue
}
info := f.info(src, i)
if info.size == 0 {
if atEOF {
// include incomplete runes
return n, true
}
return lastSegStart, true
}
// This block needs to be before the next, because it is possible to
// have an overflow for runes that are starters (e.g. with U+FF9E).
switch ss.next(info) {
case ssStarter:
lastSegStart = i
case ssOverflow:
return lastSegStart, false
case ssSuccess:
if lastCC > info.ccc {
return lastSegStart, false
}
}
if f.composing {
if !info.isYesC() {
break
}
} else {
if !info.isYesD() {
break
}
}
lastCC = info.ccc
i += int(info.size)
}
if i == n {
if !atEOF {
n = lastSegStart
}
return n, true
}
return lastSegStart, false
}
// QuickSpanString returns a boundary n such that s[0:n] == f(s[0:n]).
// It is not guaranteed to return the largest such n.
func (f Form) QuickSpanString(s string) int {
n, _ := formTable[f].quickSpan(inputString(s), 0, len(s), true)
return n
}
// FirstBoundary returns the position i of the first boundary in b
// or -1 if b contains no boundary.
func (f Form) FirstBoundary(b []byte) int {
return f.firstBoundary(inputBytes(b), len(b))
}
func (f Form) firstBoundary(src input, nsrc int) int {
i := src.skipContinuationBytes(0)
if i >= nsrc {
return -1
}
fd := formTable[f]
ss := streamSafe(0)
// We should call ss.first here, but we can't as the first rune is
// skipped already. This means FirstBoundary can't really determine
// CGJ insertion points correctly. Luckily it doesn't have to.
for {
info := fd.info(src, i)
if info.size == 0 {
return -1
}
if s := ss.next(info); s != ssSuccess {
return i
}
i += int(info.size)
if i >= nsrc {
if !info.BoundaryAfter() && !ss.isMax() {
return -1
}
return nsrc
}
}
}
// FirstBoundaryInString returns the position i of the first boundary in s
// or -1 if s contains no boundary.
func (f Form) FirstBoundaryInString(s string) int {
return f.firstBoundary(inputString(s), len(s))
}
// NextBoundary reports the index of the boundary between the first and next
// segment in b or -1 if atEOF is false and there are not enough bytes to
// determine this boundary.
func (f Form) NextBoundary(b []byte, atEOF bool) int {
return f.nextBoundary(inputBytes(b), len(b), atEOF)
}
// NextBoundaryInString reports the index of the boundary between the first and
// next segment in b or -1 if atEOF is false and there are not enough bytes to
// determine this boundary.
func (f Form) NextBoundaryInString(s string, atEOF bool) int {
return f.nextBoundary(inputString(s), len(s), atEOF)
}
func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
if nsrc == 0 {
if atEOF {
return 0
}
return -1
}
fd := formTable[f]
info := fd.info(src, 0)
if info.size == 0 {
if atEOF {
return 1
}
return -1
}
ss := streamSafe(0)
ss.first(info)
for i := int(info.size); i < nsrc; i += int(info.size) {
info = fd.info(src, i)
if info.size == 0 {
if atEOF {
return i
}
return -1
}
// TODO: Using streamSafe to determine the boundary isn't the same as
// using BoundaryBefore. Determine which should be used.
if s := ss.next(info); s != ssSuccess {
return i
}
}
if !atEOF && !info.BoundaryAfter() && !ss.isMax() {
return -1
}
return nsrc
}
// LastBoundary returns the position i of the last boundary in b
// or -1 if b contains no boundary.
func (f Form) LastBoundary(b []byte) int {
return lastBoundary(formTable[f], b)
}
func lastBoundary(fd *formInfo, b []byte) int {
i := len(b)
info, p := lastRuneStart(fd, b)
if p == -1 {
return -1
}
if info.size == 0 { // ends with incomplete rune
if p == 0 { // starts with incomplete rune
return -1
}
i = p
info, p = lastRuneStart(fd, b[:i])
if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter
return i
}
}
if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8
return i
}
if info.BoundaryAfter() {
return i
}
ss := streamSafe(0)
v := ss.backwards(info)
for i = p; i >= 0 && v != ssStarter; i = p {
info, p = lastRuneStart(fd, b[:i])
if v = ss.backwards(info); v == ssOverflow {
break
}
if p+int(info.size) != i {
if p == -1 { // no boundary found
return -1
}
return i // boundary after an illegal UTF-8 encoding
}
}
return i
}
// decomposeSegment scans the first segment in src into rb. It inserts 0x034f
// (Grapheme Joiner) when it encounters a sequence of more than 30 non-starters
// and returns the number of bytes consumed from src or iShortDst or iShortSrc.
func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
// Force one character to be consumed.
info := rb.f.info(rb.src, sp)
if info.size == 0 {
return 0
}
if s := rb.ss.next(info); s == ssStarter {
// TODO: this could be removed if we don't support merging.
if rb.nrune > 0 {
goto end
}
} else if s == ssOverflow {
rb.insertCGJ()
goto end
}
if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
return int(err)
}
for {
sp += int(info.size)
if sp >= rb.nsrc {
if !atEOF && !info.BoundaryAfter() {
return int(iShortSrc)
}
break
}
info = rb.f.info(rb.src, sp)
if info.size == 0 {
if !atEOF {
return int(iShortSrc)
}
break
}
if s := rb.ss.next(info); s == ssStarter {
break
} else if s == ssOverflow {
rb.insertCGJ()
break
}
if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
return int(err)
}
}
end:
if !rb.doFlush() {
return int(iShortDst)
}
return sp
}
// lastRuneStart returns the runeInfo and position of the last
// rune in buf or the zero runeInfo and -1 if no rune was found.
func lastRuneStart(fd *formInfo, buf []byte) (Properties, int) {
p := len(buf) - 1
for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- {
}
if p < 0 {
return Properties{}, -1
}
return fd.info(inputBytes(buf), p), p
}
// decomposeToLastBoundary finds an open segment at the end of the buffer
// and scans it into rb. Returns the buffer minus the last segment.
func decomposeToLastBoundary(rb *reorderBuffer) {
fd := &rb.f
info, i := lastRuneStart(fd, rb.out)
if int(info.size) != len(rb.out)-i {
// illegal trailing continuation bytes
return
}
if info.BoundaryAfter() {
return
}
var add [maxNonStarters + 1]Properties // stores runeInfo in reverse order
padd := 0
ss := streamSafe(0)
p := len(rb.out)
for {
add[padd] = info
v := ss.backwards(info)
if v == ssOverflow {
// Note that if we have an overflow, it the string we are appending to
// is not correctly normalized. In this case the behavior is undefined.
break
}
padd++
p -= int(info.size)
if v == ssStarter || p < 0 {
break
}
info, i = lastRuneStart(fd, rb.out[:p])
if int(info.size) != p-i {
break
}
}
rb.ss = ss
// Copy bytes for insertion as we may need to overwrite rb.out.
var buf [maxBufferSize * utf8.UTFMax]byte
cp := buf[:copy(buf[:], rb.out[p:])]
rb.out = rb.out[:p]
for padd--; padd >= 0; padd-- {
info = add[padd]
rb.insertUnsafe(inputBytes(cp), 0, info)
cp = cp[info.size:]
}
}
vendor/golang.org/x/text/unicode/norm/readwriter.go 0 → 100644
View file @ 5bee5d69
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package norm
import "io"
type normWriter struct {
rb reorderBuffer
w io.Writer
buf []byte
}
// Write implements the standard write interface. If the last characters are
// not at a normalization boundary, the bytes will be buffered for the next
// write. The remaining bytes will be written on close.
func (w *normWriter) Write(data []byte) (n int, err error) {
// Process data in pieces to keep w.buf size bounded.
const chunk = 4000
for len(data) > 0 {
// Normalize into w.buf.
m := len(data)
if m > chunk {
m = chunk
}
w.rb.src = inputBytes(data[:m])
w.rb.nsrc = m
w.buf = doAppend(&w.rb, w.buf, 0)
data = data[m:]
n += m
// Write out complete prefix, save remainder.
// Note that lastBoundary looks back at most 31 runes.
i := lastBoundary(&w.rb.f, w.buf)
if i == -1 {
i = 0
}
if i > 0 {
if _, err = w.w.Write(w.buf[:i]); err != nil {
break
}
bn := copy(w.buf, w.buf[i:])
w.buf = w.buf[:bn]
}
}
return n, err
}
// Close forces data that remains in the buffer to be written.
func (w *normWriter) Close() error {
if len(w.buf) > 0 {
_, err := w.w.Write(w.buf)
if err != nil {
return err
}
}
return nil
}
// Writer returns a new writer that implements Write(b)
// by writing f(b) to w. The returned writer may use an
// an internal buffer to maintain state across Write calls.
// Calling its Close method writes any buffered data to w.
func (f Form) Writer(w io.Writer) io.WriteCloser {
wr := &normWriter{rb: reorderBuffer{}, w: w}
wr.rb.init(f, nil)
return wr
}
type normReader struct {
rb reorderBuffer
r io.Reader
inbuf []byte
outbuf []byte
bufStart int
lastBoundary int
err error
}
// Read implements the standard read interface.
func (r *normReader) Read(p []byte) (int, error) {
for {
if r.lastBoundary-r.bufStart > 0 {
n := copy(p, r.outbuf[r.bufStart:r.lastBoundary])
r.bufStart += n
if r.lastBoundary-r.bufStart > 0 {
return n, nil
}
return n, r.err
}
if r.err != nil {
return 0, r.err
}
outn := copy(r.outbuf, r.outbuf[r.lastBoundary:])
r.outbuf = r.outbuf[0:outn]
r.bufStart = 0
n, err := r.r.Read(r.inbuf)
r.rb.src = inputBytes(r.inbuf[0:n])
r.rb.nsrc, r.err = n, err
if n > 0 {
r.outbuf = doAppend(&r.rb, r.outbuf, 0)
}
if err == io.EOF {
r.lastBoundary = len(r.outbuf)
} else {
r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf)
if r.lastBoundary == -1 {
r.lastBoundary = 0
}
}
}
}
// Reader returns a new reader that implements Read
// by reading data from r and returning f(data).
func (f Form) Reader(r io.Reader) io.Reader {
const chunk = 4000
buf := make([]byte, chunk)
rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf}
rr.rb.init(f, buf)
return rr
}
vendor/golang.org/x/text/unicode/norm/tables10.0.0.go 0 → 100644
View file @ 5bee5d69
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/golang.org/x/text/unicode/norm/tables9.0.0.go 0 → 100644
View file @ 5bee5d69
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/golang.org/x/text/unicode/norm/transform.go 0 → 100644
View file @ 5bee5d69
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package norm
import (
"unicode/utf8"
"golang.org/x/text/transform"
)
// Reset implements the Reset method of the transform.Transformer interface.
func (Form) Reset() {}
// Transform implements the Transform method of the transform.Transformer
// interface. It may need to write segments of up to MaxSegmentSize at once.
// Users should either catch ErrShortDst and allow dst to grow or have dst be at
// least of size MaxTransformChunkSize to be guaranteed of progress.
func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
n := 0
// Cap the maximum number of src bytes to check.
b := src
eof := atEOF
if ns := len(dst); ns < len(b) {
err = transform.ErrShortDst
eof = false
b = b[:ns]
}
i, ok := formTable[f].quickSpan(inputBytes(b), n, len(b), eof)
n += copy(dst[n:], b[n:i])
if !ok {
nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF)
return nDst + n, nSrc + n, err
}
if n < len(src) && !atEOF {
err = transform.ErrShortSrc
}
return n, n, err
}
func flushTransform(rb *reorderBuffer) bool {
// Write out (must fully fit in dst, or else it is an ErrShortDst).
if len(rb.out) < rb.nrune*utf8.UTFMax {
return false
}
rb.out = rb.out[rb.flushCopy(rb.out):]
return true
}
var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc}
// transform implements the transform.Transformer interface. It is only called
// when quickSpan does not pass for a given string.
func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
// TODO: get rid of reorderBuffer. See CL 23460044.
rb := reorderBuffer{}
rb.init(f, src)
for {
// Load segment into reorder buffer.
rb.setFlusher(dst[nDst:], flushTransform)
end := decomposeSegment(&rb, nSrc, atEOF)
if end < 0 {
return nDst, nSrc, errs[-end]
}
nDst = len(dst) - len(rb.out)
nSrc = end
// Next quickSpan.
end = rb.nsrc
eof := atEOF
if n := nSrc + len(dst) - nDst; n < end {
err = transform.ErrShortDst
end = n
eof = false
}
end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof)
n := copy(dst[nDst:], rb.src.bytes[nSrc:end])
nSrc += n
nDst += n
if ok {
if n < rb.nsrc && !atEOF {
err = transform.ErrShortSrc
}
return nDst, nSrc, err
}
}
}
vendor/golang.org/x/text/unicode/norm/trie.go 0 → 100644
View file @ 5bee5d69
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package norm
type valueRange struct {
value uint16 // header: value:stride
lo, hi byte // header: lo:n
}
type sparseBlocks struct {
values []valueRange
offset []uint16
}
var nfcSparse = sparseBlocks{
values: nfcSparseValues[:],
offset: nfcSparseOffset[:],
}
var nfkcSparse = sparseBlocks{
values: nfkcSparseValues[:],
offset: nfkcSparseOffset[:],
}
var (
nfcData = newNfcTrie(0)
nfkcData = newNfkcTrie(0)
)
// lookupValue determines the type of block n and looks up the value for b.
// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
// is a list of ranges with an accompanying value. Given a matching range r,
// the value for b is by r.value + (b - r.lo) * stride.
func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
offset := t.offset[n]
header := t.values[offset]
lo := offset + 1
hi := lo + uint16(header.lo)
for lo < hi {
m := lo + (hi-lo)/2
r := t.values[m]
if r.lo <= b && b <= r.hi {
return r.value + uint16(b-r.lo)*header.value
}
if b < r.lo {
hi = m
} else {
lo = m + 1
}
}
return 0
}
vendor/vendor.json
View file @ 5bee5d69
......@@ -292,10 +292,10 @@
"revisionTime": "2015-03-14T17:03:34Z"
},
{
"checksumSHA1": "2gmvVTDCks8cPhpmyDlvm0sbrXE=",
"checksumSHA1": "FYM/8R2CqS6PSNAoKl6X5gNJ20A=",
"path": "github.com/naoina/toml",
"revision": "ac014c6b6502388d89a85552b7208b8da7cfe104",
"revisionTime": "2017-04-10T21:57:17Z"
"revision": "9fafd69674167c06933b1787ae235618431ce87f",
"revisionTime": "2017-09-18T21:04:37Z"
},
{
"checksumSHA1": "xZBlSMT5o/A+EDOro6KbfHZwSNc=",
......@@ -603,6 +603,12 @@
"revision": "b4690f45fa1cafc47b1c280c2e75116efe40cc13",
"revisionTime": "2017-02-15T08:41:58Z"
},
{
"checksumSHA1": "RcrB7tgYS/GMW4QrwVdMOTNqIU8=",
"path": "golang.org/x/net/idna",
"revision": "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec",
"revisionTime": "2018-01-12T01:53:59Z"
},
{
"checksumSHA1": "7EZyXN0EmZLgGxZxK01IJua4c8o=",
"path": "golang.org/x/net/websocket",
......@@ -711,12 +717,30 @@
"revision": "85c29909967d7f171f821e7a42e7b7af76fb9598",
"revisionTime": "2017-02-11T12:01:23Z"
},
{
"checksumSHA1": "CbpjEkkOeh0fdM/V8xKDdI0AA88=",
"path": "golang.org/x/text/secure/bidirule",
"revision": "e19ae1496984b1c655b8044a65c0300a3c878dd3",
"revisionTime": "2017-12-24T20:31:28Z"
},
{
"checksumSHA1": "ziMb9+ANGRJSSIuxYdRbA+cDRBQ=",
"path": "golang.org/x/text/transform",
"revision": "85c29909967d7f171f821e7a42e7b7af76fb9598",
"revisionTime": "2017-02-11T12:01:23Z"
},
{
"checksumSHA1": "w8kDfZ1Ug+qAcVU0v8obbu3aDOY=",
"path": "golang.org/x/text/unicode/bidi",
"revision": "e19ae1496984b1c655b8044a65c0300a3c878dd3",
"revisionTime": "2017-12-24T20:31:28Z"
},
{
"checksumSHA1": "BCNYmf4Ek93G4lk5x3ucNi/lTwA=",
"path": "golang.org/x/text/unicode/norm",
"revision": "e19ae1496984b1c655b8044a65c0300a3c878dd3",
"revisionTime": "2017-12-24T20:31:28Z"
},
{
"checksumSHA1": "ikor+YKJu2eKwyFteBWhsb8IGy8=",
"path": "golang.org/x/tools/go/ast/astutil",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment