index: embarking on a most ambitious project...

This adds some lesser known extensions.

Notably, it adds php7 and php8, but not php6. Apparently,
php6 was never a thing: https://wiki.php.net/rfc/php6

PR #2263

.github/workflows/ci.yml

@@ -98,11 +98,9 @@ jobs:
         ci/macos-install-packages
 
     - name: Install Rust
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@v1
       with:
         toolchain: ${{ matrix.rust }}
-        profile: minimal
-        override: true
 
     - name: Use Cross
       if: matrix.target != ''
@@ -185,11 +183,9 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v2
     - name: Install Rust
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@v1
       with:
         toolchain: stable
-        override: true
-        profile: minimal
         components: rustfmt
     - name: Check formatting
       run: |
@@ -202,11 +198,9 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
       - name: Install Rust
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@v1
         with:
           toolchain: stable
-          profile: minimal
-          override: true
       - name: Check documentation
         env:
           RUSTDOCFLAGS: -D warnings

.github/workflows/release.yml

@@ -112,11 +112,9 @@ jobs:
         ci/macos-install-packages
 
     - name: Install Rust
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@v1
       with:
         toolchain: ${{ matrix.rust }}
-        profile: minimal
-        override: true
         target: ${{ matrix.target }}
 
     - name: Use Cross

CHANGELOG.md

@@ -1,5 +1,5 @@
-13.0.1
+TBD
-======
+===
 Unreleased changes. Release notes have not yet been written.
 
 Bug fixes:
@@ -8,6 +8,8 @@ Bug fixes:
   Fix bug when using `-w` with a regex that can match the empty string.
 * [BUG #1911](https://github.com/BurntSushi/ripgrep/issues/1911):
   Disable mmap searching in all non-64-bit environments.
+* [BUG #2236](https://github.com/BurntSushi/ripgrep/issues/2236):
+  Fix gitignore parsing bug where a trailing `\/` resulted in an error.
 
 
 13.0.0 (2021-06-12)

Cargo.lock

@@ -152,7 +152,7 @@ dependencies = [
 
 [[package]]
 name = "grep"
-version = "0.2.8"
+version = "0.2.10"
 dependencies = [
  "grep-cli",
  "grep-matcher",
@@ -179,6 +179,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "grep-index"
+version = "0.0.1"
+dependencies = [
+ "bstr 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "grep-matcher"
 version = "0.1.5"
@@ -211,7 +219,7 @@ dependencies = [
 
 [[package]]
 name = "grep-regex"
-version = "0.1.9"
+version = "0.1.10"
 dependencies = [
  "aho-corasick",
  "bstr",
@@ -224,7 +232,7 @@ dependencies = [
 
 [[package]]
 name = "grep-searcher"
-version = "0.1.8"
+version = "0.1.10"
 dependencies = [
  "bstr",
  "bytecount",

Cargo.toml

@@ -32,6 +32,7 @@ members = [
   "crates/globset",
   "crates/grep",
   "crates/cli",
+  "crates/index",
   "crates/matcher",
   "crates/pcre2",
   "crates/printer",

crates/cli/Cargo.toml

@@ -16,7 +16,7 @@ edition = "2018"
 [dependencies]
 atty = "0.2.11"
 bstr = "0.2.0"
-globset = { version = "0.4.7", path = "../globset" }
+globset = { version = "0.4.9", path = "../globset" }
 lazy_static = "1.1.0"
 log = "0.4.5"
 regex = "1.1"

crates/cli/src/decompress.rs

@@ -382,7 +382,7 @@ impl DecompressionReader {
     ///
     /// `close` is also called in `drop` as a last line of defense against
     /// resource leakage. Any error from the child process is then printed as a
-    /// warning to stderr. This can be avoided by explictly calling `close`
+    /// warning to stderr. This can be avoided by explicitly calling `close`
     /// before the CommandReader is dropped.
     pub fn close(&mut self) -> io::Result<()> {
         match self.rdr {

crates/cli/src/human.rs

@@ -8,7 +8,7 @@ use regex::Regex;
 /// An error that occurs when parsing a human readable size description.
 ///
 /// This error provides an end user friendly message describing why the
-/// description coudln't be parsed and what the expected format is.
+/// description couldn't be parsed and what the expected format is.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ParseSizeError {
     original: String,

crates/cli/src/lib.rs

@@ -212,13 +212,13 @@ pub fn is_readable_stdin() -> bool {
     !is_tty_stdin() && imp()
 }
 
-/// Returns true if and only if stdin is believed to be connectted to a tty
+/// Returns true if and only if stdin is believed to be connected to a tty
 /// or a console.
 pub fn is_tty_stdin() -> bool {
     atty::is(atty::Stream::Stdin)
 }
 
-/// Returns true if and only if stdout is believed to be connectted to a tty
+/// Returns true if and only if stdout is believed to be connected to a tty
 /// or a console.
 ///
 /// This is useful for when you want your command line program to produce
@@ -230,7 +230,7 @@ pub fn is_tty_stdout() -> bool {
     atty::is(atty::Stream::Stdout)
 }
 
-/// Returns true if and only if stderr is believed to be connectted to a tty
+/// Returns true if and only if stderr is believed to be connected to a tty
 /// or a console.
 pub fn is_tty_stderr() -> bool {
     atty::is(atty::Stream::Stderr)

crates/cli/src/process.rs

@@ -221,7 +221,7 @@ impl CommandReader {
     ///
     /// `close` is also called in `drop` as a last line of defense against
     /// resource leakage. Any error from the child process is then printed as a
-    /// warning to stderr. This can be avoided by explictly calling `close`
+    /// warning to stderr. This can be avoided by explicitly calling `close`
     /// before the CommandReader is dropped.
     pub fn close(&mut self) -> io::Result<()> {
         // Dropping stdout closes the underlying file descriptor, which should

crates/core/args.rs

@@ -104,7 +104,7 @@ struct ArgsImp {
     ///
     /// It's important that this is only built once, since building this goes
     /// through regex compilation and various types of analyses. That is, if
-    /// you need many of theses (one per thread, for example), it is better to
+    /// you need many of these (one per thread, for example), it is better to
     /// build it once and then clone it.
     matcher: PatternMatcher,
     /// The paths provided at the command line. This is guaranteed to be

crates/core/subject.rs

@@ -67,7 +67,7 @@ impl SubjectBuilder {
         if subj.is_file() {
             return Some(subj);
         }
-        // We got nothin. Emit a debug message, but only if this isn't a
+        // We got nothing. Emit a debug message, but only if this isn't a
         // directory. Otherwise, emitting messages for directories is just
         // noisy.
         if !subj.is_dir() {

crates/globset/README.md

@@ -19,7 +19,7 @@ Add this to your `Cargo.toml`:
 
 ```toml
 [dependencies]
-globset = "0.3"
+globset = "0.4"
 ```
 
 ### Features
@@ -78,12 +78,12 @@ assert_eq!(set.matches("src/bar/baz/foo.rs"), vec![0, 2]);
 
 This crate implements globs by converting them to regular expressions, and
 executing them with the
-[`regex`](https://github.com/rust-lang-nursery/regex)
+[`regex`](https://github.com/rust-lang/regex)
 crate.
 
 For single glob matching, performance of this crate should be roughly on par
 with the performance of the
-[`glob`](https://github.com/rust-lang-nursery/glob)
+[`glob`](https://github.com/rust-lang/glob)
 crate. (`*_regex` correspond to benchmarks for this library while `*_glob`
 correspond to benchmarks for the `glob` library.)
 Optimizations in the `regex` crate may propel this library past `glob`,
@@ -108,7 +108,7 @@ test many_short_glob      ... bench:       1,063 ns/iter (+/- 47)
 test many_short_regex_set ... bench:         186 ns/iter (+/- 11)
 ```
 
-### Comparison with the [`glob`](https://github.com/rust-lang-nursery/glob) crate
+### Comparison with the [`glob`](https://github.com/rust-lang/glob) crate
 
 * Supports alternate "or" globs, e.g., `*.{foo,bar}`.
 * Can match non-UTF-8 file paths correctly.

crates/grep/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "grep"
-version = "0.2.8"  #:version
+version = "0.2.10"  #:version
 authors = ["Andrew Gallant <jamslam@gmail.com>"]
 description = """
 Fast line oriented regex searching as a library.
@@ -18,8 +18,8 @@ grep-cli = { version = "0.1.6", path = "../cli" }
 grep-matcher = { version = "0.1.5", path = "../matcher" }
 grep-pcre2 = { version = "0.1.5", path = "../pcre2", optional = true }
 grep-printer = { version = "0.1.6", path = "../printer" }
-grep-regex = { version = "0.1.9", path = "../regex" }
+grep-regex = { version = "0.1.10", path = "../regex" }
-grep-searcher = { version = "0.1.8", path = "../searcher" }
+grep-searcher = { version = "0.1.10", path = "../searcher" }
 
 [dev-dependencies]
 termcolor = "1.0.4"

crates/ignore/Cargo.toml

@@ -20,7 +20,7 @@ bench = false
 
 [dependencies]
 crossbeam-utils = "0.8.0"
-globset = { version = "0.4.7", path = "../globset" }
+globset = { version = "0.4.9", path = "../globset" }
 lazy_static = "1.1"
 log = "0.4.5"
 memchr = "2.1"

crates/ignore/src/default_types.rs

@@ -57,6 +57,7 @@ pub const DEFAULT_TYPES: &[(&str, &[&str])] = &[
     ("dhall", &["*.dhall"]),
     ("diff", &["*.patch", "*.diff"]),
     ("docker", &["*Dockerfile*"]),
+    ("dts", &["*.dts", "*.dtsi"]),
     ("dvc", &["Dvcfile", "*.dvc"]),
     ("ebuild", &["*.ebuild"]),
     ("edn", &["*.edn"]),
@@ -145,9 +146,9 @@ pub const DEFAULT_TYPES: &[(&str, &[&str])] = &[
     ]),
     ("mako", &["*.mako", "*.mao"]),
     ("man", &["*.[0-9lnpx]", "*.[0-9][cEFMmpSx]"]),
-    ("markdown", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
+    ("markdown", &["*.markdown", "*.md", "*.mdown", "*.mkd", "*.mkdn"]),
     ("matlab", &["*.m"]),
-    ("md", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
+    ("md", &["*.markdown", "*.md", "*.mdown", "*.mkd", "*.mkdn"]),
     ("meson", &["meson.build", "meson_options.txt"]),
     ("minified", &["*.min.html", "*.min.css", "*.min.js"]),
     ("mint", &["*.mint"]),
@@ -166,7 +167,12 @@ pub const DEFAULT_TYPES: &[(&str, &[&str])] = &[
     ("pascal", &["*.pas", "*.dpr", "*.lpr", "*.pp", "*.inc"]),
     ("pdf", &["*.pdf"]),
     ("perl", &["*.perl", "*.pl", "*.PL", "*.plh", "*.plx", "*.pm", "*.t"]),
-    ("php", &["*.php", "*.php3", "*.php4", "*.php5", "*.phtml"]),
+    ("php", &[
+        // note that PHP 6 doesn't exist
+        // See: https://wiki.php.net/rfc/php6
+        "*.php", "*.php3", "*.php4", "*.php5", "*.php7", "*.php8",
+        "*.pht", "*.phtml"
+    ]),
     ("po", &["*.po"]),
     ("pod", &["*.pod"]),
     ("postscript", &["*.eps", "*.ps"]),

crates/ignore/src/gitignore.rs

@@ -474,10 +474,13 @@ impl GitignoreBuilder {
         }
         // If it ends with a slash, then this should only match directories,
         // but the slash should otherwise not be used while globbing.
-        if let Some((i, c)) = line.char_indices().rev().nth(0) {
+        if line.as_bytes().last() == Some(&b'/') {
-            if c == '/' {
+            glob.is_only_dir = true;
-                glob.is_only_dir = true;
+            line = &line[..line.len() - 1];
-                line = &line[..i];
+            // If the slash was escaped, then remove the escape.
+            // See: https://github.com/BurntSushi/ripgrep/issues/2236
+            if line.as_bytes().last() == Some(&b'\\') {
+                line = &line[..line.len() - 1];
             }
         }
         glob.actual = line.to_string();

crates/index/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+publish = false
+name = "grep-index"
+version = "0.0.1"  #:version
+authors = ["Andrew Gallant <jamslam@gmail.com>"]
+description = """
+Grep, but with an index.
+"""
+documentation = "https://docs.rs/grep-index"
+homepage = "https://github.com/BurntSushi/ripgrep"
+repository = "https://github.com/BurntSushi/ripgrep"
+readme = "README.md"
+keywords = ["regex", "grep", "search", "index", "ngram"]
+license = "Unlicense/MIT"
+edition = "2018"
+
+[dependencies]
+bstr = "0.2"
+regex-syntax = "0.6.14"

crates/index/LICENSE-MIT

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Andrew Gallant
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

crates/index/README.md

@@ -0,0 +1,8 @@
+grep-index
+----------
+WIP.
+
+[![Build status](https://github.com/BurntSushi/ripgrep/workflows/ci/badge.svg)](https://github.com/BurntSushi/ripgrep/actions)
+[![](https://img.shields.io/crates/v/grep-regex.svg)](https://crates.io/crates/grep-index)
+
+Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).

crates/index/UNLICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

crates/index/src/lib.rs

@@ -0,0 +1,7 @@
+/*!
+TODO
+*/
+
+#![allow(warnings)]
+
+mod literal;

crates/printer/src/json.rs

@@ -147,7 +147,7 @@ impl JSONBuilder {
 /// is not limited to UTF-8 exclusively, which in turn implies that matches
 /// may be reported that contain invalid UTF-8. Moreover, this printer may
 /// also print file paths, and the encoding of file paths is itself not
-/// guarnateed to be valid UTF-8. Therefore, this printer must deal with the
+/// guaranteed to be valid UTF-8. Therefore, this printer must deal with the
 /// presence of invalid UTF-8 somehow. The printer could silently ignore such
 /// things completely, or even lossily transcode invalid UTF-8 to valid UTF-8
 /// by replacing all invalid sequences with the Unicode replacement character.

crates/printer/src/standard.rs

@@ -1594,7 +1594,7 @@ impl<'a, M: Matcher, W: WriteColor> StandardImpl<'a, M, W> {
     /// multiple lines.
     ///
     /// Note that this doesn't just return whether the searcher is in multi
-    /// line mode, but also checks if the mater can match over multiple lines.
+    /// line mode, but also checks if the matter can match over multiple lines.
     /// If it can't, then we don't need multi line handling, even if the
     /// searcher has multi line mode enabled.
     fn multi_line(&self) -> bool {

crates/printer/src/summary.rs

@@ -508,7 +508,7 @@ impl<'p, 's, M: Matcher, W: WriteColor> SummarySink<'p, 's, M, W> {
     /// multiple lines.
     ///
     /// Note that this doesn't just return whether the searcher is in multi
-    /// line mode, but also checks if the mater can match over multiple lines.
+    /// line mode, but also checks if the matter can match over multiple lines.
     /// If it can't, then we don't need multi line handling, even if the
     /// searcher has multi line mode enabled.
     fn multi_line(&self, searcher: &Searcher) -> bool {

crates/regex/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "grep-regex"
-version = "0.1.9"  #:version
+version = "0.1.10"  #:version
 authors = ["Andrew Gallant <jamslam@gmail.com>"]
 description = """
 Use Rust's regex library with the 'grep' crate.

crates/regex/src/config.rs

@@ -175,6 +175,36 @@ impl ConfiguredHIR {
         self.config.crlf && self.expr.is_line_anchored_end()
     }
 
+    /// Returns the line terminator configured on this expression.
+    ///
+    /// When we have beginning/end anchors (NOT line anchors), the fast line
+    /// searching path isn't quite correct. Or at least, doesn't match the
+    /// slow path. Namely, the slow path strips line terminators while the
+    /// fast path does not. Since '$' (when multi-line mode is disabled)
+    /// doesn't match at line boundaries, the existence of a line terminator
+    /// might cause it to not match when it otherwise would with the line
+    /// terminator stripped.
+    ///
+    /// Since searching with text anchors is exceptionally rare in the
+    /// context of line oriented searching (multi-line mode is basically
+    /// always enabled), we just disable this optimization when there are
+    /// text anchors. We disable it by not returning a line terminator, since
+    /// without a line terminator, the fast search path can't be executed.
+    ///
+    /// See: <https://github.com/BurntSushi/ripgrep/issues/2260>
+    pub fn line_terminator(&self) -> Option<LineTerminator> {
+        if self.is_any_anchored() {
+            None
+        } else {
+            self.config.line_terminator
+        }
+    }
+
+    /// Returns true if and only if the underlying HIR has any text anchors.
+    fn is_any_anchored(&self) -> bool {
+        self.expr.is_any_anchored_start() || self.expr.is_any_anchored_end()
+    }
+
     /// Builds a regular expression from this HIR expression.
     pub fn regex(&self) -> Result<Regex, Error> {
         self.pattern_to_regex(&self.expr.to_string())

crates/regex/src/matcher.rs

@@ -52,8 +52,12 @@ impl RegexMatcherBuilder {
 
         let matcher = RegexMatcherImpl::new(&chir)?;
         log::trace!("final regex: {:?}", matcher.regex());
+        let mut config = self.config.clone();
+        // We override the line terminator in case the configured expr doesn't
+        // support it.
+        config.line_terminator = chir.line_terminator();
         Ok(RegexMatcher {
-            config: self.config.clone(),
+            config,
             matcher,
             fast_line_regex,
             non_matching_bytes,

crates/searcher/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "grep-searcher"
-version = "0.1.8"  #:version
+version = "0.1.10"  #:version
 authors = ["Andrew Gallant <jamslam@gmail.com>"]
 description = """
 Fast line oriented regex searching as a library.
@@ -23,7 +23,7 @@ log = "0.4.5"
 memmap = { package = "memmap2", version = "0.5.3" }
 
 [dev-dependencies]
-grep-regex = { version = "0.1.9", path = "../regex" }
+grep-regex = { version = "0.1.10", path = "../regex" }
 regex = "1.1"
 
 [features]

crates/searcher/src/searcher/glue.rs

@@ -1512,4 +1512,31 @@ and exhibited clearly, with a label attached.\
             )
             .unwrap();
     }
+
+    // See: https://github.com/BurntSushi/ripgrep/issues/2260
+    #[test]
+    fn regression_2260() {
+        use grep_regex::RegexMatcherBuilder;
+
+        use crate::SearcherBuilder;
+
+        let matcher = RegexMatcherBuilder::new()
+            .line_terminator(Some(b'\n'))
+            .build(r"^\w+$")
+            .unwrap();
+        let mut searcher = SearcherBuilder::new().line_number(true).build();
+
+        let mut matched = false;
+        searcher
+            .search_slice(
+                &matcher,
+                b"GATC\n",
+                crate::sinks::UTF8(|_, _| {
+                    matched = true;
+                    Ok(true)
+                }),
+            )
+            .unwrap();
+        assert!(matched);
+    }
 }

tests/regression.rs

@@ -1118,3 +1118,11 @@ pipc () {  # [-h] [-U|-u <pkgspec>[,<pkgspec>...]] [<reqs-in>...] [-- <pip-compi
     let expected = " [-h] [-U|-u <pkgspec>[,<pkgspec>...]] [<reqs-in>...] [-- <pip-compile-arg>...]\n";
     eqnice!(expected, cmd.stdout());
 });
+
+// See: https://github.com/BurntSushi/ripgrep/issues/2236
+rgtest!(r2236, |dir: Dir, mut cmd: TestCommand| {
+    dir.create(".ignore", r"foo\/");
+    dir.create_dir("foo");
+    dir.create("foo/bar", "test\n");
+    cmd.args(&["test"]).assert_err();
+});