[perf] Remove memory copy when using string delimiter

This commit is contained in:
Junegunn Choi
2016-08-14 04:23:37 +09:00
parent ddc7bb9064
commit d9c8a9a880
3 changed files with 67 additions and 6 deletions

View File

@@ -140,13 +140,13 @@ func Tokenize(text util.Chars, delimiter Delimiter) []Token {
return withPrefixLengths(tokens, prefixLength)
}
var tokens []string
if delimiter.str != nil {
tokens = strings.Split(text.ToString(), *delimiter.str)
for i := 0; i < len(tokens)-1; i++ {
tokens[i] = tokens[i] + *delimiter.str
}
} else if delimiter.regex != nil {
return withPrefixLengths(text.Split(*delimiter.str), 0)
}
// FIXME performance
var tokens []string
if delimiter.regex != nil {
str := text.ToString()
for len(str) > 0 {
loc := delimiter.regex.FindStringIndex(str)