searcher: always strip BOM

This fixes a bug where a BOM prefix was included. While this was somewhat
intentional in order to have a faithful "UTF8 passthru" option, in
practice, this causes problems such as breaking patterns like `^` in a
really non-obvious way.

The actual fix was to add a new API to encoding_rs_io, which this commit
brings in.

Fixes #1163
This commit is contained in:
Andrew Gallant
2019-01-25 17:18:57 -05:00
parent 9a9f54d44c
commit 276e2c9b9a
4 changed files with 14 additions and 4 deletions

View File

@@ -15,7 +15,7 @@ license = "Unlicense/MIT"
[dependencies]
bytecount = "0.5"
encoding_rs = "0.8.14"
encoding_rs_io = "0.1.3"
encoding_rs_io = "0.1.4"
grep-matcher = { version = "0.1.1", path = "../grep-matcher" }
log = "0.4.5"
memchr = "2.1"

View File

@@ -307,6 +307,7 @@ impl SearcherBuilder {
decode_builder
.encoding(self.config.encoding.as_ref().map(|e| e.0))
.utf8_passthru(true)
.strip_bom(true)
.bom_override(true);
Searcher {
config: config,