Fix delimiter regex to properly support caret (^)

Fix #2861
This commit is contained in:
Junegunn Choi
2022-07-21 21:21:06 +09:00
parent ccc4677252
commit 0d06c28b19
2 changed files with 21 additions and 8 deletions

View File

@@ -156,14 +156,14 @@ func Tokenize(text string, delimiter Delimiter) []Token {
// FIXME performance
var tokens []string
if delimiter.regex != nil {
for len(text) > 0 {
loc := delimiter.regex.FindStringIndex(text)
if len(loc) < 2 {
loc = []int{0, len(text)}
}
last := util.Max(loc[1], 1)
tokens = append(tokens, text[:last])
text = text[last:]
locs := delimiter.regex.FindAllStringIndex(text, -1)
begin := 0
for _, loc := range locs {
tokens = append(tokens, text[begin:loc[1]])
begin = loc[1]
}
if begin < len(text) {
tokens = append(tokens, text[begin:])
}
}
return withPrefixLengths(tokens, 0)