twin/ripgrep
1
0
Fork 0
mirror of https://github.com/BurntSushi/ripgrep.git synced 2025-07-27 18:21:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: twin:grep-pcre2-0.1.3
Branches Tags
twin:master twin:ag/rollup twin:ag/fix-ci twin:ag/index twin:ag/fix-multiline-replace-duplicate twin:ag/fix-cve-2021-3013 twin:ag/moarci twin:ag/ghactions twin:ag/snafu twin:ag/disable-bom-sniff twin:ag/partial-bstr-migration twin:ag/bstr-migration twin:ag/prepare-0.10.0 twin:ag/libripgrep-freeze-2 twin:ag/libripgrep-freeze-1 twin:ag/changelog
twin:globset-0.4.16 twin:14.1.1 twin:14.1.0 twin:ignore-0.4.22 twin:14.0.3 twin:14.0.2 twin:grep-0.3.1 twin:grep-printer-0.2.1 twin:grep-searcher-0.1.13 twin:14.0.1 twin:grep-0.3.0 twin:grep-printer-0.2.0 twin:14.0.0 twin:grep-0.2.13 twin:grep-searcher-0.1.12 twin:grep-pcre2-0.1.7 twin:grep-regex-0.1.12 twin:grep-matcher-0.1.7 twin:grep-cli-0.1.10 twin:ignore-0.4.21 twin:globset-0.4.14 twin:globset-0.4.13 twin:globset-0.4.12 twin:grep-cli-0.1.9 twin:globset-0.4.11 twin:grep-cli-0.1.8 twin:grep-0.2.12 twin:ignore-0.4.20 twin:grep-0.2.11 twin:grep-printer-0.1.7 twin:grep-searcher-0.1.11 twin:grep-pcre2-0.1.6 twin:grep-regex-0.1.11 twin:grep-matcher-0.1.6 twin:grep-cli-0.1.7 twin:ignore-0.4.19 twin:globset-0.4.10 twin:grep-0.2.10 twin:grep-searcher-0.1.10 twin:grep-0.2.9 twin:grep-searcher-0.1.9 twin:grep-regex-0.1.10 twin:globset-0.4.9 twin:globset-0.4.8 twin:13.0.0 twin:grep-0.2.8 twin:grep-printer-0.1.6 twin:grep-searcher-0.1.8 twin:grep-pcre2-0.1.5 twin:grep-regex-0.1.9 twin:grep-matcher-0.1.5 twin:grep-cli-0.1.6 twin:ignore-0.4.18 twin:globset-0.4.7 twin:ignore-0.4.17 twin:globset-0.4.6 twin:12.1.1 twin:grep-0.2.7 twin:grep-cli-0.1.5 twin:ignore-0.4.16 twin:12.1.0 twin:grep-0.2.6 twin:grep-printer-0.1.5 twin:grep-regex-0.1.8 twin:ignore-0.4.15 twin:12.0.1 twin:ignore-0.4.14 twin:ignore-0.4.13 twin:grep-regex-0.1.7 twin:12.0.0 twin:grep-searcher-0.1.7 twin:grep-regex-0.1.6 twin:grep-printer-0.1.4 twin:grep-pcre2-0.1.4 twin:grep-matcher-0.1.4 twin:ignore-0.4.12 twin:grep-0.2.5 twin:globset-0.4.5 twin:grep-cli-0.1.4 twin:ignore-0.4.11 twin:grep-regex-0.1.5 twin:grep-searcher-0.1.6 twin:ignore-0.4.10 twin:11.0.2 twin:grep-matcher-0.1.3 twin:grep-regex-0.1.4 twin:ignore-0.4.9 twin:ignore-0.4.8 twin:grep-searcher-0.1.5 twin:grep-printer-0.1.3 twin:grep-cli-0.1.3 twin:globset-0.4.4 twin:11.0.1 twin:11.0.0 twin:grep-0.2.4 twin:grep-cli-0.1.2 twin:grep-printer-0.1.2 twin:grep-searcher-0.1.4 twin:grep-regex-0.1.3 twin:grep-pcre2-0.1.3 twin:grep-matcher-0.1.2 twin:ignore-0.4.7 twin:globset-0.4.3 twin:grep-regex-0.1.2 twin:grep-searcher-0.1.3 twin:grep-searcher-0.1.2 twin:ignore-0.4.6 twin:ignore-0.4.5 twin:grep-0.2.3 twin:grep-pcre2-0.1.2 twin:0.10.0 twin:grep-searcher-0.1.1 twin:globset-0.4.2 twin:grep-pcre2-0.1.1 twin:grep-0.2.2 twin:grep-printer-0.1.1 twin:ignore-0.4.4 twin:grep-regex-0.1.1 twin:grep-matcher-0.1.1 twin:grep-cli-0.1.1 twin:grep-0.2.1 twin:grep-cli-0.1.0 twin:grep-0.2.0 twin:grep-search-0.1.0 twin:grep-matcher-0.1.0 twin:grep-printer-0.1.0 twin:grep-pcre2-0.1.0 twin:grep-regex-0.1.0 twin:0.9.0 twin:grep-0.1.9 twin:ignore-0.4.3 twin:globset-0.4.1 twin:ignore-0.4.2 twin:globset-0.4.0 twin:termcolor-0.3.6 twin:0.8.1 twin:termcolor-0.3.5 twin:ignore-0.4.1 twin:0.8.0 twin:ignore-0.4.0 twin:globset-0.3.0 twin:termcolor-0.3.4 twin:wincolor-0.1.6 twin:grep-0.1.8 twin:wincolor-0.1.5 twin:0.7.1 twin:ignore-0.3.1 twin:ignore-0.3.0 twin:grep-0.1.7 twin:globset-0.2.1 twin:0.7.0 twin:termcolor-0.3.3 twin:0.6.0 twin:ignore-0.2.2 twin:ignore-0.2.1 twin:wincolor-0.1.4 twin:0.5.2 twin:ignore-0.1.9 twin:0.5.1 twin:termcolor-0.3.2 twin:0.5.0 twin:grep-0.1.6 twin:ignore-0.1.8 twin:globset-0.1.4 twin:termcolor-0.3.1 twin:wincolor-0.1.3 twin:termcolor-0.3.0 twin:wincolor-0.1.2 twin:0.4.0 twin:0.3.2 twin:ignore-0.1.6 twin:0.3.1 twin:termcolor-0.1.1 twin:0.3.0 twin:termcolor-0.1.0 twin:wincolor-0.1.0 twin:0.2.9 twin:ignore-0.1.5 twin:0.2.8 twin:0.2.7 twin:ignore-0.1.4 twin:grep-0.1.4 twin:globset-0.1.2 twin:0.2.6 twin:ignore-0.1.3 twin:ignore-0.1.2 twin:0.2.5 twin:0.2.4 twin:ignore-0.1.1 twin:ignore-0.1.0 twin:globset-0.1.1 twin:0.2.3 twin:0.2.2 twin:globset-0.1.0 twin:0.2.1 twin:0.2.0 twin:0.1.18 twin:0.1.17 twin:0.1.16 twin:0.1.15 twin:0.1.14 twin:0.1.13 twin:0.1.12 twin:0.1.11 twin:0.1.10 twin:0.1.9 twin:0.1.8 twin:0.1.7 twin:0.1.6 twin:0.1.5 twin:0.1.4 twin:0.1.3 twin:0.1.2 twin:0.1.1 twin:0.1.0 twin:0.0.19 twin:0.0.18 twin:0.0.17 twin:0.0.16 twin:0.0.15 twin:0.0.14 twin:0.0.13 twin:0.0.12 twin:0.0.11 twin:0.0.10 twin:0.0.9 twin:0.0.8 twin:0.0.7 twin:0.0.6 twin:0.0.5 twin:0.0.4 twin:0.0.3 twin:0.0.2 twin:0.0.1
...
compare: twin:grep-matcher-0.1.4
Branches Tags
twin:ag/rollup twin:master twin:ag/fix-ci twin:ag/index twin:ag/fix-multiline-replace-duplicate twin:ag/fix-cve-2021-3013 twin:ag/moarci twin:ag/ghactions twin:ag/snafu twin:ag/disable-bom-sniff twin:ag/partial-bstr-migration twin:ag/bstr-migration twin:ag/prepare-0.10.0 twin:ag/libripgrep-freeze-2 twin:ag/libripgrep-freeze-1 twin:ag/changelog
twin:globset-0.4.16 twin:14.1.1 twin:14.1.0 twin:ignore-0.4.22 twin:14.0.3 twin:14.0.2 twin:grep-0.3.1 twin:grep-printer-0.2.1 twin:grep-searcher-0.1.13 twin:14.0.1 twin:grep-0.3.0 twin:grep-printer-0.2.0 twin:14.0.0 twin:grep-0.2.13 twin:grep-searcher-0.1.12 twin:grep-pcre2-0.1.7 twin:grep-regex-0.1.12 twin:grep-matcher-0.1.7 twin:grep-cli-0.1.10 twin:ignore-0.4.21 twin:globset-0.4.14 twin:globset-0.4.13 twin:globset-0.4.12 twin:grep-cli-0.1.9 twin:globset-0.4.11 twin:grep-cli-0.1.8 twin:grep-0.2.12 twin:ignore-0.4.20 twin:grep-0.2.11 twin:grep-printer-0.1.7 twin:grep-searcher-0.1.11 twin:grep-pcre2-0.1.6 twin:grep-regex-0.1.11 twin:grep-matcher-0.1.6 twin:grep-cli-0.1.7 twin:ignore-0.4.19 twin:globset-0.4.10 twin:grep-0.2.10 twin:grep-searcher-0.1.10 twin:grep-0.2.9 twin:grep-searcher-0.1.9 twin:grep-regex-0.1.10 twin:globset-0.4.9 twin:globset-0.4.8 twin:13.0.0 twin:grep-0.2.8 twin:grep-printer-0.1.6 twin:grep-searcher-0.1.8 twin:grep-pcre2-0.1.5 twin:grep-regex-0.1.9 twin:grep-matcher-0.1.5 twin:grep-cli-0.1.6 twin:ignore-0.4.18 twin:globset-0.4.7 twin:ignore-0.4.17 twin:globset-0.4.6 twin:12.1.1 twin:grep-0.2.7 twin:grep-cli-0.1.5 twin:ignore-0.4.16 twin:12.1.0 twin:grep-0.2.6 twin:grep-printer-0.1.5 twin:grep-regex-0.1.8 twin:ignore-0.4.15 twin:12.0.1 twin:ignore-0.4.14 twin:ignore-0.4.13 twin:grep-regex-0.1.7 twin:12.0.0 twin:grep-searcher-0.1.7 twin:grep-regex-0.1.6 twin:grep-printer-0.1.4 twin:grep-pcre2-0.1.4 twin:grep-matcher-0.1.4 twin:ignore-0.4.12 twin:grep-0.2.5 twin:globset-0.4.5 twin:grep-cli-0.1.4 twin:ignore-0.4.11 twin:grep-regex-0.1.5 twin:grep-searcher-0.1.6 twin:ignore-0.4.10 twin:11.0.2 twin:grep-matcher-0.1.3 twin:grep-regex-0.1.4 twin:ignore-0.4.9 twin:ignore-0.4.8 twin:grep-searcher-0.1.5 twin:grep-printer-0.1.3 twin:grep-cli-0.1.3 twin:globset-0.4.4 twin:11.0.1 twin:11.0.0 twin:grep-0.2.4 twin:grep-cli-0.1.2 twin:grep-printer-0.1.2 twin:grep-searcher-0.1.4 twin:grep-regex-0.1.3 twin:grep-pcre2-0.1.3 twin:grep-matcher-0.1.2 twin:ignore-0.4.7 twin:globset-0.4.3 twin:grep-regex-0.1.2 twin:grep-searcher-0.1.3 twin:grep-searcher-0.1.2 twin:ignore-0.4.6 twin:ignore-0.4.5 twin:grep-0.2.3 twin:grep-pcre2-0.1.2 twin:0.10.0 twin:grep-searcher-0.1.1 twin:globset-0.4.2 twin:grep-pcre2-0.1.1 twin:grep-0.2.2 twin:grep-printer-0.1.1 twin:ignore-0.4.4 twin:grep-regex-0.1.1 twin:grep-matcher-0.1.1 twin:grep-cli-0.1.1 twin:grep-0.2.1 twin:grep-cli-0.1.0 twin:grep-0.2.0 twin:grep-search-0.1.0 twin:grep-matcher-0.1.0 twin:grep-printer-0.1.0 twin:grep-pcre2-0.1.0 twin:grep-regex-0.1.0 twin:0.9.0 twin:grep-0.1.9 twin:ignore-0.4.3 twin:globset-0.4.1 twin:ignore-0.4.2 twin:globset-0.4.0 twin:termcolor-0.3.6 twin:0.8.1 twin:termcolor-0.3.5 twin:ignore-0.4.1 twin:0.8.0 twin:ignore-0.4.0 twin:globset-0.3.0 twin:termcolor-0.3.4 twin:wincolor-0.1.6 twin:grep-0.1.8 twin:wincolor-0.1.5 twin:0.7.1 twin:ignore-0.3.1 twin:ignore-0.3.0 twin:grep-0.1.7 twin:globset-0.2.1 twin:0.7.0 twin:termcolor-0.3.3 twin:0.6.0 twin:ignore-0.2.2 twin:ignore-0.2.1 twin:wincolor-0.1.4 twin:0.5.2 twin:ignore-0.1.9 twin:0.5.1 twin:termcolor-0.3.2 twin:0.5.0 twin:grep-0.1.6 twin:ignore-0.1.8 twin:globset-0.1.4 twin:termcolor-0.3.1 twin:wincolor-0.1.3 twin:termcolor-0.3.0 twin:wincolor-0.1.2 twin:0.4.0 twin:0.3.2 twin:ignore-0.1.6 twin:0.3.1 twin:termcolor-0.1.1 twin:0.3.0 twin:termcolor-0.1.0 twin:wincolor-0.1.0 twin:0.2.9 twin:ignore-0.1.5 twin:0.2.8 twin:0.2.7 twin:ignore-0.1.4 twin:grep-0.1.4 twin:globset-0.1.2 twin:0.2.6 twin:ignore-0.1.3 twin:ignore-0.1.2 twin:0.2.5 twin:0.2.4 twin:ignore-0.1.1 twin:ignore-0.1.0 twin:globset-0.1.1 twin:0.2.3 twin:0.2.2 twin:globset-0.1.0 twin:0.2.1 twin:0.2.0 twin:0.1.18 twin:0.1.17 twin:0.1.16 twin:0.1.15 twin:0.1.14 twin:0.1.13 twin:0.1.12 twin:0.1.11 twin:0.1.10 twin:0.1.9 twin:0.1.8 twin:0.1.7 twin:0.1.6 twin:0.1.5 twin:0.1.4 twin:0.1.3 twin:0.1.2 twin:0.1.1 twin:0.1.0 twin:0.0.19 twin:0.0.18 twin:0.0.17 twin:0.0.16 twin:0.0.15 twin:0.0.14 twin:0.0.13 twin:0.0.12 twin:0.0.11 twin:0.0.10 twin:0.0.9 twin:0.0.8 twin:0.0.7 twin:0.0.6 twin:0.0.5 twin:0.0.4 twin:0.0.3 twin:0.0.2 twin:0.0.1

168 Commits
grep-pcre2 ... grep-match

Author SHA1 Message Date
Andrew Gallant
92daa34eb3 ripgrep: release 12.0.0 2020-03-15 21:42:54 -04:00
Andrew Gallant
a8c1fb7c88 changelog: prepare for 12.0.0 release 2020-03-15 21:06:45 -04:00
Andrew Gallant
52ec68799c ci: make script names consistent 2020-03-15 21:06:45 -04:00
Andrew Gallant
c0d78240df ci: remove Travis and appveyor specific stuff 2020-03-15 21:06:45 -04:00
Andrew Gallant
cda9acb876 ci: rebuild release infrastructure on GitHub Actions 2020-03-15 21:06:45 -04:00
Andrew Gallant
1ece50694e readme: update file size 2020-03-15 13:27:31 -04:00
Andrew Gallant
f3a966bcbc readme: add 'Unicode' label to ugrep 2020-03-15 13:26:02 -04:00
Andrew Gallant
a38913b63a readme: update benchmarks 
This also updates the corpora used, so previous times (and counts) are
not comparable.

We also remove some tools, likt pt, sift and ucg, since they appear to
be no longer maintained. ag isn't really maintained either, but it still
has significant mind share, so we retain a benchmark for it.

We also upgrade ack to version 3, and remove the clarification on how
`-w` is implemented.

We also add `git grep -P` (uses PCRE2) which appears to be much faster
than `git grep -E`.

Finally, we add ugrep which is a new up and comer in this space.

Fixes #1474
 2020-03-15 13:21:18 -04:00
Andrew Gallant
e772a95b58 regex: avoid using literal optimizations when whitespace is detected 
If a literal is entirely whitespace, then it's quite likely that it is
very common. So when that case occurs, just don't do (inner) literal
optimizations at all.

The regex engine may still make sub-optimal decisions here, but that's a
problem for another day.

Fixes #1087
 2020-03-15 13:19:14 -04:00
Andrew Gallant
9dd4bf8d7f style: fix rust-analyzer lint warnings 2020-03-15 13:19:14 -04:00
Andrew Gallant
c4c43c733e cli: add --no-ignore-files flag 
The purpose of this flag is to force ripgrep to ignore all --ignore-file
flags (whether they come before or after --no-ignore-files).

This flag can be overridden with --ignore-files.

Fixes #1466
 2020-03-15 13:19:14 -04:00
Andrew Gallant
447506ebe0 doc: clarify globing behavior 
Fixes #1442, Fixes #1478
 2020-03-15 13:19:14 -04:00
Andrew Gallant
12e4180985 doc: remove CPU features from man pages 
It doesn't really belong in the man page since it's an artifact of a
build/runtime configuration. Moreover, it inhibits reproducible builds.

Fixes #1441
 2020-03-15 13:19:14 -04:00
Andrew Gallant
daa8319398 doc: note ripgrep's stdin behavior 
Fixes #1439
 2020-03-15 13:19:14 -04:00
pierrenn
3a6a24a52a cli: add engine flag 
This permits switching between the different regex engine modes that
ripgrep supports. The purpose of this flag is to make it easier to
extend ripgrep with additional regex engines.

Closes #1488, Closes #1502
 2020-03-15 09:30:58 -04:00
pierrenn
aab3d80374 args: refactor to permit adding other engines 
This is in preparation for adding a new --engine flag which is intended
to eventually supplant --auto-hybrid-regex.

While there are no immediate plans to add more regex engines to ripgrep,
this is intended to make it easier to maintain a patch to ripgrep with
an additional regex engine. See #1488 for more details.
 2020-03-15 09:24:28 -04:00
Andrew Gallant
1856cda77b style: fix rust-analyzer lints in core 2020-03-15 09:04:54 -04:00
Andrew Gallant
7340d8dbbe deps: update everything 
This adds one new dependency, maybe-uninit, which is brought in by
crossbeam-channel[1]. This is to apparently fix some unsound code
without bumping the MSRV. Since ripgrep uses the latest stable release
of Rust, the maybe-uninit crate should compile down to nothing and just
re-export std's `MaybeUninit` type.

[1] - https://github.com/crossbeam-rs/crossbeam/pull/458
 2020-03-15 08:32:33 -04:00
chip
50d2047ae2 crates: update URLs in Cargo.toml 
This corrects an oversight when the repo was re-organized to
have its crates moved into a 'crates' sub-directory.

PR #1505
 2020-02-28 20:31:43 -05:00
Wolf Honore
227436624f ignore/types: add coq type 
PR #1504
 2020-02-28 19:11:29 -05:00
pierreN
5bfdd3a652 ci: fix ci by removing fetch-depth 1 
It's not clear why removing this makes things work. I've submitted
PRs that passed CI with fetch-depth=1. Maybe it only fails when
PRs are submitted from external contributors?

Either way, for now, we remove this and absorb the extra cost in
order to get PRs passing CI again.

PR #1501
 2020-02-27 08:53:06 -05:00
Andrew Gallant
ecec6147d1 doc: be more vague in the FAQ 
The existing vagueness was not enough to prevent people from lawyering
me over it.
 2020-02-22 09:13:31 -05:00
Lucien Greathouse
db7a8cdcb5 globset: Implement serde::{Serialize, Deserialize} for Glob 
PR #1492
 2020-02-21 07:40:47 -05:00
Andrew Gallant
eef7a7e7ff readme: update CI badge 2020-02-20 18:15:15 -05:00
Andrew Gallant
4176050cdd ignore: another simplification 
Again, thanks to @zsugabubus!
 2020-02-20 17:26:34 -05:00
Andrew Gallant
109460fce2 ignore: simplify parallel worker initialization 
We can just ask the channel whether any work has been loaded. Normally
querying a channel for its length is a strong predictor of bugs, but in
this case, we do it before we ever attempt a `recv`, so it should work.

Kudos to @zsugabubus for suggesting this!
 2020-02-20 16:50:41 -05:00
Andrew Gallant
da3431b478 ci: switch build to GitHub Actions 2020-02-20 16:07:51 -05:00
Andrew Gallant
f314b0d55f ignore: fix parallel traversal 
It turns out that the previous version wasn't quite correct. Namely, it
was possible for the following sequence to occur:

1. Consider that all workers, except for one, are `waiting`.
2. The last remaining worker finds one more job to do and sends it on
   the channel.
3. One of the previously `waiting` workers wakes up from the job that
   the last running worker sent, but `self.resume()` has not been
   called yet.
4. The last worker, from (2), calls `get_work` and sees that the
   channel has nothing on it, so it executes `self.waiting() ==
   1`. Since the worker in (3) hasn't called `self.resume()` yet,
   `self.waiting() == 1` evaluates to true.
5. This sets off a chain reaction that stops all workers, despite that
   fact that (3) got more work (which could itself spawn more work).

The end result is that the traversal may terminate while their are still
outstanding work items to process. This problem was observed through
spurious failures in CI. I was not actually able to reproduce the bug
locally.

We fix this by changing our strategy to detect termination using a
counter. Namely, we increment the counter just before sending new work
and decrement the counter just after finishing work. In this way, we
guarantee that the counter only ever reaches 0 once there is no more
work to process.

See #1337 for more discussion. Many thanks to @zsugabubus for helping me
work through this.
 2020-02-20 16:07:51 -05:00
Andrew Gallant
fab5c812f3 tests: add debugging output 
The transient failures appear to be persisting and they are quite
difficult to debug. So include a full directory listing in the output of
every test failure.
 2020-02-20 16:07:51 -05:00
Andrew Gallant
c824d095a7 tests: use std::env::consts::EXE_SUFFIX 
This avoids a conditional compilation knob and is likely more portable.
 2020-02-20 16:07:51 -05:00
Andrew Gallant
ee21897ebd tests: make 'cross test' work 
The reason why it wasn't working was the integration tests. Namely, the
integration tests attempted to execute the 'rg' binary directly from
inside cross's docker container. But this obviously doesn't work when
'rg' was compiled for a totally different architecture.

Cross normally does this by hooking into the Rust test infrastructure
and causing tests to run with 'qemu'. But our integration tests didn't
do that. This commit fixes our test setup to check for cross's
environment variable that points to the 'qemu' binary. Once we have
that, we just use 'qemu-foo rg' instead of 'rg'. Piece of cake.
 2020-02-20 16:07:51 -05:00
Andrew Gallant
0373f6ddb0 ci: soft-disable Travis and AppVeyor 2020-02-20 16:07:51 -05:00
asymmetric
b44554c803 ignore/types: add K type 
Adds support for files used by the K executable semantic framework:
http://www.kframework.org/index.php/Main_Page

PR #1493
 2020-02-19 07:07:09 -05:00
Andrew Gallant
0874aa115c repo: make ripgrep build with the new organization 2020-02-17 19:24:53 -05:00
Andrew Gallant
fdd8510fdd repo: move all source code in crates directory 
The top-level listing was just getting a bit too long for my taste. So
put all of the code in one directory and shrink the large top-level mess
to a small top-level mess.

NOTE: This commit only contains renames. The subsequent commit will
actually make ripgrep build again. We do it this way with the naive hope
that this will make it easier for git history to track the renames.
Sigh.
 2020-02-17 19:24:53 -05:00
Andrew Gallant
0bc4f0447b style: rustfmt everything 
This is why I was so intent on clearing the PR queue. This will
effectively invalidate all existing patches, so I wanted to start from a
clean slate.

We do make one little tweak: we put the default type definitions in
their own file and tell rustfmt to keep its grubby mits off of it. We
also sort it lexicographically and hopefully will enforce that from here
on.
 2020-02-17 19:24:53 -05:00
Andrew Gallant
c95f29e3ba ci: check rustfmt in Travis 2020-02-17 19:24:53 -05:00
Andrew Gallant
3644208b03 ci: set MSRV to Rust 1.41.0 
The next release will be ripgrep 12, so we bump to the latest stable
release of Rust.
 2020-02-17 19:24:53 -05:00
Andrew Gallant
66f045e055 changelog: add commit links 
... now that we have stable identifiers.
 2020-02-17 17:34:19 -05:00
zsugabubus
3d59bd98aa ignore: rework inter-thread messaging 
Change the meaning of `Quit` message. Now it means terminate. The final
"dance" is unnecessary, because by the time quitting begins, no thread
will ever spawn a new `Work`. The trick was to replace the heuristic
spin-loop with blocking receive.

Closes #1337
 2020-02-17 17:16:28 -05:00
Andrew Gallant
52d7f47420 ignore: treat symbolic links to directories as directories 
Due to how walkdir works if symlinks are not followed, symlinks to
directories are seen as simple files by ripgrep. This caused a panic
in some cases due to receiving a WalkEvent::Exit event without a
corresponding WalkEvent::Dir event.

This is fixed by looking at the metadata of the file in the case of a
symlink to determine if it's a directory. We are careful to only do
this stat check when the depth of the entry is 0, as this bug only
impacts us when 1) we aren't following symlinks generally and 2) the
user provides a symlinked directory that we do follow as a top-level
path to search.

Fixes #1389, Closes #1397
 2020-02-17 17:16:28 -05:00
Andrew Gallant
75cbe88fa2 cli: add --no-unicode, deprecate --no-pcre2-unicode 
This adds a universal --no-unicode flag that is intended to work for all
supported regex engines. There is no point in retaining
--no-pcre2-unicode, so we make them aliases to the new flags and
deprecate them.
 2020-02-17 17:16:28 -05:00
Andrew Gallant
711426a632 cli: add --no-require-git flag 
This flag prevents ripgrep from requiring one to search a git repository
in order to respect git-related ignore rules (global, .gitignore and
local excludes). This actually corresponds to behavior ripgrep had long
ago, but #934 changed that. It turns out that users were relying on this
buggy behavior. In most cases, fixing it as simple as converting one's
rules to .ignore or .rgignore files. Unfortunately, there are other use
cases---like Perforce automatically respecting .gitignore files---that
make a strong case for ripgrep to at least support this.

The UX of a flag like this is absolutely atrocious. It's so obscure that
it's really not worth explicitly calling it out anywhere. Moreover, the
error cases that occur when this flag isn't used (but its behavior is
desirable) will not be intuitive, do not seem easily detectable and will
not guide users to this flag. Nevertheless, the motivation for this is
just barely strong enough for me to begrudgingly accept this.

Fixes #1414, Closes #1416
 2020-02-17 17:16:28 -05:00
Andrew Gallant
01eeec56bb deb: fix fish completion install location 
It looks like `completions` is owned by Fish itself. Third party
completions should go in `vendor_completions.d`.

Fixes #1485
 2020-02-17 17:16:28 -05:00
Jakub Wieczorek
322fc75a3d ignore: make walker visit untraversable directories 
This commit fixes an inconsistency between the serial and the parallel
directory walkers around visiting a directory for which the user holds
insufficient permissions to descend into.

The serial walker does produce a successful entry for a directory that
it cannot descend into due to insufficient permissions. However, before
this change that has not been the case for the parallel walker, which
would produce an `Err` item not only when descending into a directory
that it cannot read from but also for the directory entry itself.

This change brings the behaviour of the parallel variant in line with
that of the serial one.

Fixes #1346, Closes #1365
 2020-02-17 17:16:28 -05:00
Jakub Wieczorek
b435eaafc8 grep-regex: fix inner literal extraction bug 
This appears to be another transcription bug from copying this code from
the prefix literal detection from inside the regex crate. Namely, when
it comes to inner literals, we only want to treat counted repetition as
two separate cases: the case when the minimum match is 0 and the case
when the minimum match is more than 0. In the former case, we treat
`e{0,n}` as `e*` and in the latter we treat `e{m,n}` where `m >= 1` as
just `e`.

We could definitely do better here. e.g., This means regexes like
`(foo){10}` will only have `foo` extracted as a literal, where searching
for the full literal would likely be faster.

The actual bug here was that we were not implementing this logic
correctly. Namely, we weren't always "cutting" the literals in the
second case to prevent them from being expanded.

Fixes #1319, Closes #1367
 2020-02-17 17:16:28 -05:00
Ed Page
f8e70294d5 ignore: allow post-processing at end-of-thread 
On top of the parallel-walk's closures, this provides a Visitor API.
This clarifies the role of the two different closures in the `run`
API and allows implementing of `Drop` for post-processing once traversal
is finished.

The closure API is maintained not just for compatibility but also
convinience for simple cases.

Fixes #469, Closes #1430
 2020-02-17 17:16:28 -05:00
Ed Page
578e2d47a8 core: simplify parallel walking using borrows 
This changes ripgrep to use ignore's new support for borrowing data when
walking in parallel.
 2020-02-17 17:16:28 -05:00
Ed Page
9f7c2ebc09 ignore: allow parallel walker to borrow data 
This makes it so the caller can more easily refactor from
single-threaded to multi-threaded walking. If they want to support both,
this makes it easier to do so with a single initialization code-path. In
particular, it side-steps the need to put everything into an `Arc`.

This is not a breaking change because it strictly increases the number
of allowed inputs to `WalkParallel::run`.

Closes #1410, Closes #1432
 2020-02-17 17:16:28 -05:00
Andrew Gallant
5c1eac41a3 changelog: highlight a bad performance regression 2020-02-17 17:16:28 -05:00
Johannes Altmanninger
6f2b79f584 ignore: use git commondir for sourcing .git/info/exclude 
Git looks for this file in GIT_COMMON_DIR, which is usually the same
as GIT_DIR (.git). However, when searching inside a linked worktree,
.git is usually a file that contains the path of the actual git dir,
which in turn contains a file "commondir" which references the directory
where info/exclude may reside, alongside other configuration shared across
all worktrees. This directory is usually the git dir of the main worktree.

Unlike git this does *not* read environment variables GIT_DIR and
GIT_COMMON_DIR, because it is not clear how to interpret them when
searching multiple repositories.

Fixes #1445, Closes #1446
 2020-02-17 17:16:28 -05:00
Andrew Gallant
0c3b673e4c cli: make ripgrep work in non-existent directories 
It turns out that querying the CWD while in a directory that no longer
exists results in an error. Since the CWD is queried every time ripgrep
starts---whether it needs it or not---for dealing with glob matching,
ripgrep winds up being completely useless inside a non-existent
directory.

We fix this in a few different ways:

* Firstly, if std::env::current_dir() fails, then we fall back to trying
  to read the `PWD` environment variable.
* If that fails, that we return a more sensible error message so that a
  user can at least react to the problem. Previously, the error message
  was inscrutable.
* Finally, we try to avoid the problem altogether by building empty glob
  matchers if not globs were provided, thus side-stepping querying the
  CWD completely.

Fixes #1291, Closes #1400
 2020-02-17 17:16:28 -05:00
Naveen Nathan
297b428c8c cli: add --no-ignore-exclude flag 
This commit adds a new --no-ignore-exclude flag that permits disabling
the use of .git/info/exclude filtering. Local exclusions are manual
configurations to a repository and are not shared, so it is sometimes
useful to disable to get a consistent view of a repository.

This also adds a new section to the man page that describes automatic
filtering.

Closes #1420
 2020-02-17 17:16:28 -05:00
Manfred Endres
804b43ecd8 globset: implement FromStr for Glob 
The `globset::Glob` type [`new`] function creates a new value with an
`&str` parameter which returns an `Result<Glob, Error>` object. This is
exactly what [`std::str::FromStr::from_str`][`std::str::FromStr`] defines.
Libraries like [`clap`] use [`std::str::FromStr`] to create objects from
provided commandline arguments. This change makes this library usable
without a newtype wrapper.

[`std::str::FromStr`]: 	https://doc.rust-lang.org/std/str/trait.FromStr.html
[`clap`]:		https://docs.rs/clap/2.33.0/clap/macro.value_t.html
[`new`]:		https://docs.rs/globset/0.4.4/globset/struct.Glob.html#method.new

Closes #1447
 2020-02-17 17:16:28 -05:00
Lucien Greathouse
2263b8ac92 globset: add GlobMatcher::glob 
This exposes the underlying `Glob` used to compile the matcher. This can
be useful for wrapping up the glob matcher in other types.

Closes #1454
 2020-02-17 17:16:28 -05:00
Andrew Gallant
cd8ec38a68 grep-regex: add fast path for -w/--word-regexp 
Previously, ripgrep would always defer to the regex engine's capturing
matches in order to implement word matching. Namely, ripgrep would
determine the correct match offsets via a capturing group, since the
word regex is itself generated from the user supplied regex.

Unfortunately, the regex engine's capturing mode is still fairly slow,
so this commit adds a fast path to avoid capturing mode in the vast
majority of cases. See comments in the code for details.
 2020-02-17 17:16:28 -05:00
Andrew Gallant
6a0e0147e0 grep-regex: improve literal detection with -w 
When the -w/--word-regexp was used, ripgrep would in many cases fail to
apply literal optimizations. This occurs specifically when the regex
given by the user is an alternation of literals with no common prefixes
or suffixes, e.g.,

    rg -w 'foo|bar|baz|quux'

In this case, the inner literal detector fails. Normally, this would
result in literal prefixes being detected by the regex engine. But
because of the -w/--word-regexp flag, the actual regex that we run ends
up looking like this:

    (^|\W)(foo|bar|baz|quux)($|\W)

which of course defeats any prefix or suffix literal optimizations in
the regex crate's somewhat naive extractor. (A better extractor could
still do literal optimizations in the above case.)

So this commit fixes this by falling back to prefix or suffix literals
when they're available instead of prematurely giving up and assuming the
regex engine will do the rest.
 2020-02-17 17:16:28 -05:00
Andrew Gallant
ad97e9c93f grep-regex: improve inner literal detection 
This fixes an interesting performance bug where the inner literal
extractor would sometimes choose a sub-optimal literal. For example,
consider the regex:

    \x20+Sherlock Holmes\x20+

(The `\x20` is the ASCII code for a space character, which we use here
to just make it clearer. It otherwise does not matter.)

Previously, this would see the initial \x20 and then stop collecting
literals after the `+` repetition operator. This was because the inner
literal detector was adapter from the prefix literal detector, which had
to stop here. Namely, while \x20S would be a valid prefix (for example),
\x20\x20S would also be a valid prefix. As would \x20\x20\x20S and so
on. So the prefix detector would have to stop at the repetition
operator. Otherwise, only searching for \x20S could potentially scan
farther then the starting position of the next match.

However, for inner literals, this calculus no longer makes sense. We can
freely search for, e.g., \x20S without missing matches that start with
\x20\x20S precisely because we know this is an inner literal which may
not correspond to the start of a match.

With this fix, the literal that is now detected is

    \x20Sherlock Holmes\x20

Which is much better. We achieve this by no longer "cutting" literals
after seeing a `+` repetition operator. Instead, we permit literals to
continue to be extended.

The reason why this is important is because using \x20 as the literal to
search for is generally bad juju since it is so common. In fact, we
should probably add more logic here to either avoid such things or give
up entirely on the inner literal optimization if it detected a literal
that we think is very common. But we punt on such things here.
 2020-02-17 17:16:28 -05:00
Robert Irelan
24f8a3e5ec doc: document all file type 
This adds it to the guide and the docs for the --type flag.

Fixes #1344, Closes #1472
 2020-02-17 17:16:28 -05:00
Mikko Vedru
1bdb767851 doc: improve docs for --sort and --sortr flags 
I improved the help documentation in the following manner and for the
following reasons:

1. It's only logical to put the default sub-option on the first possible
line, as well as to separately mention that it is indeed the default
sub-option.

2. Additional options for the flags should describe the main points of
their purpose without requiring user to read the whole help entry. In my
opinion, the information sub-options' influence on multi-threading and
speed are important enough to warrant their inclusion in each
sub-option's description line text.

Closes #1434
 2020-02-17 17:16:28 -05:00
Andreas Stieger
a4897eca23 readme: simplify openSUSE instructions 
Closes #1436
 2020-02-17 17:16:28 -05:00
Collin Styles
a070722ff2 cli: add --include-zero flag 
This flag, when used in conjunction with --count or --count-matches,
will print a result for each file searched even if there were zero
matches in that file. This is off by default but can be enabled to make
ripgrep behave more like grep.

This also clarifies some of the defaults for the
grep-printer::SummaryBuilder type.

Closes #1370, Closes #1405
 2020-02-17 17:16:28 -05:00
Matěj Cepl
4628d77808 ignore/types: add spec file type 
This is for RPM package SPEC files.

Fixes #946, Closes #1449
 2020-02-17 17:16:28 -05:00
Ximin Luo
f8418c6a52 explicitly declare lazy_static dependency 
`benches/bench.rs` uses lazy_static but Cargo.toml does not declare a
dependency on it. This causes rustc to use its own internal private
copy instead. Sometimes this causes unintuitive errors like this Debian
bug: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=942243

The underlying issue is https://github.com/rust-lang/rust#27812 but it
can be avoided by explicitly declaring the dependency, which you are
supposed to do anyways.

Closes #1435
 2020-02-17 17:16:28 -05:00
luh2
040ca45ba0 ignore/types: add xhtml to xml file type 
Closes #1426
 2020-02-17 17:16:28 -05:00
Andrew Gallant
91470572cd changelog: add notes about new file types 2020-02-17 17:16:28 -05:00
Sven-Hendrik Haase
027adbf485 ignore/types: add 'diff' file type 
This includes .patch and .diff files.

Fixes #1418, Closes #1419
 2020-02-17 17:16:28 -05:00
Mohammad AlSaleh
e71eedf0eb cli: add --no-context-separator flag 
--context-separator='' still adds a new line separator, which could
still potentially be useful. So we add a new `--no-context-separator`
flag that completely disables context separators even when the -A/-B/-C
context flags are used.

Closes #1390
 2020-02-17 17:16:28 -05:00
Andrew Gallant
88f46d12f1 tests: remove existing test directory 
I'm surprised this wasn't caught until now, but if a test directory
already exists, then it was reused. This can result in hard to debug
problems with tests when, e.g., file names are changed and a recursive
search is executed.
 2020-02-17 17:16:28 -05:00
sharkdp
a18cf6ec39 ignore: add existence check for ignore files 
This commit adds a simple `.exists()` check for `.gitignore`,
`.ignore`, and other similar files before actually calling
`File::open(…)` in `GitIgnoreBuilder::add`.

The reason is that a simple existence check via `stat` can be faster
than actually trying to `open` the file, see
https://stackoverflow.com/a/12774387/704831. As we typically expect(?)
the number of directories *without* ignore files to be much larger
than the number of directories *with* ignore files, this leads to an
overall speedup.

The performance gain is not huge for `rg`, but can be quite significant
if more `.gitignore`-like files are added via
`add_custom_ignore_filename`. The speedup is *larger* for folders with
*low* files-per-directory ratios.

Note though that we do not do this check on Windows until a specific
analysis there suggests this is beneficial. Namely, Windows generally
has slower file system operations, so it's not clear whether this
speculative check is actually a benefit or not.

Benchmark results
-----------------

`rg --files` in my home folder (200k results, 6.5 files per directory):

| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|---:|---:|---:|---:|
| `./rg-master --files` | 396.4 ± 3.2 | 390.9 | 400.0 | 1.05 |
| `./rg-feature --files` | 376.0 ± 3.6 | 369.3 | 383.5 | 1.00 |

`rg --files --hidden` in my home folder (800k results, 5.4
files per directory)

| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `./rg-master --files --hidden` | 1.575 ± 0.012 | 1.560 | 1.597 | 1.06 |
| `./rg-feature --files --hidden` | 1.479 ± 0.011 | 1.464 | 1.496 | 1.00 |

`rg --files` in the chromium-79.0.3915.2 source tree (300k results, 12.7 files per
directory)

| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|---:|---:|---:|---:|
| `~/rg-master --files` | 445.2 ± 5.3 | 435.6 | 453.0 | 1.04 |
| `~/rg-feature --files` | 428.9 ± 7.0 | 418.2 | 440.0 | 1.00 |

`rg --files` in the linux-5.3 source tree (65k results, 15.1
files per directory)

| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|---:|---:|---:|---:|
| `./rg-master --files` | 94.5 ± 1.9 | 89.8 | 98.5 | 1.02 |
| `./rg-feature --files` | 92.6 ± 2.7 | 88.4 | 98.7 | 1.00 |

Closes #1381
 2020-02-17 17:16:28 -05:00
Gibson Fahnestock
c78c3236a8 readme: remove outdated SIMD info 
Looks like the upstream brew Formula [0][] now has SIMD support, so
remove the extraneous info now that the custom tap is no longer needed
[1][].

[0]: https://github.com/Homebrew/homebrew-core/blob/master/Formula/ripgrep.rb
[1]: f3083e4574

PR #1431
 2020-02-15 17:19:22 -05:00
Sorin Sbarnea
7cf21600cd readme: document CentOS 8 support 
ripgrep install instructions are valid even for the 7 version. The tool
works without problems on these too.

PR #1428
 2020-02-15 17:16:57 -05:00
Jonathan Mast
647b0d3977 ignore/types: add HAML and ERB 
These are commonly used templating languages for Ruby, add their
extensions to the filetypes list for convenient filtering.

PR #1407
 2020-02-15 09:18:32 -05:00
Jeff S
e572fc1683 ignore/types: add slim, slime, and skim templates 
PR #1391
 2020-02-15 09:17:46 -05:00
Andrew Gallant
9cb93abd11 ignore: allow use of Error::description 
We can remove it in the next semver incompatible release.
 2020-02-10 06:44:21 -05:00
Luca Kredel
41695c66fa ignore/types: add typoscript file type 
Add the file types for TypoScript - the configuration language of the
TYPO3 CMS.

PR #1477
 2020-02-07 08:41:00 -05:00
Andrew Gallant
cb0dfda936 faq: add section about donations 
This is asked often enough that it's worth having a canonical answer.
 2020-02-05 13:09:11 -05:00
Andrew Gallant
74d1fe59e9 deps: update everything 2020-01-30 18:33:40 -05:00
Andrew Gallant
9fd1e202e0 deps: update regex, regex-syntax and aho-corasick 
Notably, this brings in a bug fix reported by @okdana:
https://github.com/rust-lang/regex/issues/640
 2020-01-30 18:32:56 -05:00
Robert Irelan
e76807b1b5 ignore/types: add *.org_archive to org file type 
.org_archive is the default extension for Org archive files, created when
entries from an Org-mode file are archived (see
<https://orgmode.org/org.html#Moving-subtrees>). These files are still in Org
mode format, so it's worth searching them at the same time as non-archive Org
mode files.

PR #1475
 2020-01-29 13:59:34 -05:00
Andrew Gallant
f8fb65f7e3 globset: fix benchmarks 
There were apparently a lot of unused things, including lazy_static.
 2020-01-27 16:45:12 -05:00
Tristan Waddington
98de8d248a ignore/types: make 'gradle' it's own type 
This change maintains the existing behavior of the 'groovy' type, which
includes both .groovy and .gradle files.

PR #1470
 2020-01-23 06:51:11 -05:00
Crestwave
c358700dfb readme: add instructions for Haiku x86_64 and x86_gcc2 
PR #1465
 2020-01-21 07:34:24 -05:00
Alex Touchet
8670a4a969 readme: update outdated links 
PR #1463
 2020-01-21 07:32:54 -05:00
Oliver Newman
e3b1f86908 doc: add missing "will" to the user guide 
PR #1462
 2020-01-20 17:26:08 -05:00
Jan Verbeek
46b07bb2ee ignore/types: fix postscript globs 
The postscript globs were missing asterisks, so they were treated as
literal filenames.

PR #1461
 2020-01-20 07:48:57 -05:00
Andrew Gallant
8bdf84e3a8 deps: update everything 2020-01-16 19:47:23 -05:00
Andrew Gallant
5a6e17fcc1 deps: various updates 
Most of these updates (sans thread_local) are from crates I maintain
that have seen updates recently.

Notably, this includes a bump to `termcolor 1.1.0` which includes
support for respecting `NO_COLOR`. This commit therefore means that
ripgrep now supports `NO_COLOR`.

As an added bonus, we drop a dependency on Windows. (Although the total
amount of code compiled remains the same.)

Closes #1186
 2020-01-11 10:09:10 -05:00
Andrew Gallant
00bfcd14a6 ignore-0.4.11 2020-01-10 15:08:27 -05:00
Andrew Gallant
bf0ddc4675 ci: fix musl docker build 
Looks like the old japaric images are bunk. We update our docker image
to be based on the new rustembedded images and configure cross to use
it.

Turns out that this wasn't due to a stale docker image, but rather, a
bug in cross: https://github.com/rust-embedded/cross/issues/357
We work around that bug by installing the master branch of cross. Sigh.
 2020-01-10 15:07:47 -05:00
Andrew Gallant
0fb3f6a159 ci: disable github actions for now 
The CI build failures are annoying and distracting. Hopefully soon I'll
be able to invest more time in the switch.
 2020-01-10 15:07:47 -05:00
Andrew Gallant
837fb5e21f deps: update to crossbeam-channel 0.4 
Closes #1427
 2020-01-10 15:07:47 -05:00
Andrew Gallant
2e1815606e deps: update to bytecount 0.6 
Looks like there aren't any major changes other than dependency updates.
 2020-01-10 15:07:47 -05:00
Andrew Gallant
cb2f6ddc61 deps: update to thread_local 1.0 
We also update the pcre2 and regex dependencies, which removes any other
lingering uses of thread_local 0.3.
 2020-01-10 15:07:47 -05:00
Andrew Gallant
bd7a42602f deps: bump to base64 0.11 2020-01-10 15:07:47 -05:00
Andrew Gallant
528ce56e1b deps: run cargo update 
The only new dependency is an unused target specific dependency hermit
via the atty crate.
 2020-01-10 15:07:47 -05:00
Yevgen Antymyrov
8892bf648c doc: fix typo in FAQ 2019-09-25 08:13:27 -04:00
Jonathan Clem
8cb7271b64 ci: get GitHub Actions running again 
Basically, matrix.os needs to be defined for every build. We
were commenting out some of the builds in order to debug
CI in the `include` section, but we also need to comment them
out in the `build section.
 2019-09-11 09:08:24 -04:00
Andrew Gallant
4858267f3b ci: initial github actions config 2019-08-31 09:24:44 -04:00
Andrew Gallant
5011dba2fd ignore: remove unused parameter 2019-08-28 20:21:34 -04:00
Andrew Gallant
e14f9195e5 deps: update everything 2019-08-28 20:18:47 -04:00
Andrew Gallant
ef0e7af56a deps: update bstr to 0.2.7 
The new bstr release contains a small performance bug fix where some
trivial methods weren't being inlined.
 2019-08-11 10:41:05 -04:00
Todd Walton
b266818aa5 doc: use XDG_CONFIG_HOME in comments 
XDG_CONFIG_DIR does not actually exist.

PR #1347
 2019-08-09 13:37:37 -04:00
LawAbidingCactus
81415ae52d doc: update to reflect glob matching behavior change 
Specifically, paths contains a `/` are not allowed to match any
other slash in the path, even as a prefix. So `!.git` is the correct
incantation for ignoring a `.git` directory that occurs anywhere 
in the path.
 2019-08-07 13:47:18 -04:00
Andrew Gallant
5c4584aa7c grep-regex-0.1.5 2019-08-06 09:51:13 -04:00
Andrew Gallant
0972c6e7c7 grep-searcher-0.1.6 2019-08-06 09:50:52 -04:00
Andrew Gallant
0a372bf2e4 deps: update ignore 2019-08-06 09:50:35 -04:00
Andrew Gallant
345124a7fa ignore-0.4.10 2019-08-06 09:47:45 -04:00
Andrew Gallant
31807f805a deps: drop tempfile 
We were only using it to create temporary directories for `ignore`
tests, but it pulls in a bunch of dependencies and we don't really need
randomness. So just use our own simple wrapper instead.
 2019-08-06 09:46:05 -04:00
Andrew Gallant
4de227fd9a deps: update everything 
Mostly this just updates regex and its assorted dependencies. This does
drop utf8-ranges and ucd-util, in accordance with changes to
regex-syntax and regex.
 2019-08-05 13:50:55 -04:00
jimbo1qaz
d7ce274722 readme: Debian Buster is stable now 
PR #1338
 2019-08-04 08:06:10 -04:00
Andrew Gallant
5b10328f41 changelog: update with bug fix 2019-08-02 07:37:27 -04:00
Andrew Gallant
813c676eca searcher: fix roll buffer bug 
This commit fixes a subtle bug in how the line buffer was rolling its
contents. Specifically, when ripgrep searches without memory maps,
it uses a "roll" buffer for incremental line oriented search without
needing to read the entire file into memory at once. The roll buffer
works by reading a chunk of bytes from the file into memory, and then
searching everything in that buffer up to the last `\n` byte. The bytes
*after* the last `\n` byte are preserved, since they likely correspond
to *part* of the next line. Once ripgrep is done searching the buffer,
it "rolls" the buffer such that the start of the next line is at the
beginning of the buffer, and then ripgrep reads more data into the
buffer starting at the (possibly) partial end of that line.

The implication of this strategy, necessarily so, is that a buffer must
be big enough to fit a single line in memory. This is because the regex
engine needs a contiguous block of memory to search, so there is no way
to search anything smaller than a single line. So if a file contains a
single line with 7.5 million bytes, then the buffer will grow to be at
least that size. (Many files have super long lines like this, but they
tend to be *binary* files, which ripgrep will detect and stop searching
unless the user forces it with the `-a/--text` flag. So in practice,
they aren't usually a problem. However, in this case, #1335 found a case
where a plain text file had a line with 7.5 million bytes.)

Now, for performance reasons, ripgrep reuses these buffers across its
search. Typically, it will create `N` of these line buffers when it
starts (where `N` is the number of threads it is using), and then reuse
them without creating any new ones as it searches through files.

This means that if you search a file with a very long line, that buffer
will expand to be big enough to store that line. ripgrep never contracts
these buffers, so once it searches the next file, ripgrep will continue
to use this large buffer. While it might be prudent to contract these
buffers in some circumstances, this isn't otherwise inherently a
problem. The memory has already been allocated, and there isn't much
cost to using it, other than the fact that ripgrep hangs on to it and
never gives it back to the OS.

However, the `roll` implementation described above had a really
important bug in it that was impacted by the size of the buffer.
Specifically, it used the following to "roll" the partial line at the
end of the buffer to the beginning:

    self.buf.copy_within_str(self.pos.., 0);

Which means that if the buffer is very large, ripgrep will copy
*everything* from `self.pos` (which might be very small, e.g., for small
files) to the end of the buffer, and move it to the beginning of the
buffer. This will happen repeatedly each time the buffer is used to
search small files, which winds up being quite a large slow down if the
line was exceptionally large (say, megabytes).

It turns out that copying everything is completely unnecessary. We only
need to copy the remainder of the last read to the beginning of the
buffer. Everything *after* the last read in the buffer is just free
space that can be filled for the next read. So, all we need to do is
copy just those bytes:

    self.buf.copy_within_str(self.pos..self.end, 0);

... which is typically much much smaller than the rest of the buffer.

This was likely also causing small performance losses in other cases as
well. For example, when searching a lot of small files, ripgrep would
likely do a lot more copying than necessary. Although, given that the
default buffer size is 8KB, this extra copying was likely pretty small,
and was thus harder to observe.

Fixes #1335
 2019-08-02 07:23:27 -04:00
Andrew Gallant
f625d72b6f pkg: update brew tap to 11.0.2 2019-08-01 19:39:53 -04:00
Andrew Gallant
3de31f7527 ci: fix musl deployment 
The docker image that the Linux binary is now built in does not have
ASCII doc installed, so setup Cross to point to my own image with those
tools installed.
 2019-08-01 18:41:44 -04:00
Andrew Gallant
e402d6c260 ripgrep: release 11.0.2 2019-08-01 18:02:15 -04:00
Andrew Gallant
48b5bdc441 src: remove old directories 
termcolor has had its own repository for a while now. No need for these
redirects any more.
 2019-08-01 17:49:28 -04:00
Andrew Gallant
709ca91f50 ignore: release 0.4.9 2019-08-01 17:48:37 -04:00
Andrew Gallant
9c220f9a9b grep-regex: release 0.1.4 2019-08-01 17:47:45 -04:00
Andrew Gallant
9085bed139 grep-matcher: release 0.1.3 2019-08-01 17:46:59 -04:00
Andrew Gallant
931ab35f76 changelog: start work on 11.0.2 release 2019-08-01 17:42:38 -04:00
Andrew Gallant
b5e5979ff1 deps: update everything 
This drops `spin` and `autocfg`, yay.
 2019-08-01 17:42:38 -04:00
Andrew Gallant
052c857da0 doc: mention .ignore and .rgignore more prominently 
Fixes #1284
 2019-08-01 17:37:46 -04:00
Andrew Gallant
5e84e784c8 doc: add translations section 
We note that they may not be up to date and are unofficial.

Fixes #1246
 2019-08-01 17:37:46 -04:00
Andrew Gallant
01e8e11621 doc: improve PCRE2 failure mode documentation 
If a user tries to search for an explicit `\n` character in a PCRE2
regex, ripgrep won't report an error and instead will (likely) silently
fail to match.

Fixes #1261
 2019-08-01 17:32:44 -04:00
Ninan John
9268ff8e8d ripgrep: fix bug when CWD has directory named - 
Specifically, when searching stdin, if the current directory has a
directory named `-`, then the `--with-filename` flag would automatically
be turned on. This is because `--with-filename` is automatically enabled
when ripgrep is given a single path that is a directory. When ripgrep is
given empty arguments, and if it is searching stdin, then its default
path list is just simple `["-"]`. The `is_dir` check passes, and
`--with-filename` gets enabled.

This commit fixes the problem by checking whether the path is `-` first.
If so, then we assume it isn't a directory. This is fine, since if it is
a directory and one asks to search it explicitly, then ripgrep will
interpret `-` as stdin anyway (which is arguably a bug on its own, but
probably not one worth fixing).

Fixes #1223, Closes #1292
 2019-08-01 17:27:23 -04:00
dana
c2cb0a4de4 ripgrep: add --glob-case-insensitive 
This flag forces -g/--glob patterns to be treated case-insensitively, as with
--iglob patterns.

Fixes #1293
 2019-08-01 17:08:58 -04:00
Andrew Gallant
adb9332f52 regex: fix -F aho-corasick optimization 
It turns out that when the -F flag was used, if any of the patterns
contained a regex meta character (such as `.`), then we winded up
escaping the pattern first before handing it off to Aho-Corasick, which
treats all patterns literally.

We continue to apply band-aides here and just avoid Aho-Corasick if
there is an escape in any of the literal patterns. This is unfortunate,
but making this work better requires more refactoring, and the right
solution is to get this optimization pushed down into the regex engine.

Fixes #1334
 2019-08-01 16:58:12 -04:00
Matthew Davidson
bc37c32717 ignore/types: add edn type from Clojure ecosystem 
PR #1330
 2019-07-29 16:43:28 -04:00
Andrew Gallant
08ae4da2b7 deps: update them 
There are some nice removals. It looks like rand has slimmed down, and
smallvec is gone now as well.
 2019-07-25 07:52:33 -04:00
Andrew Gallant
7ac95c1f50 deps: bump ignore 2019-07-24 12:56:47 -04:00
Andrew Gallant
7a6903bd4e ignore-0.4.8 2019-07-24 12:56:01 -04:00
Tiziano Santoro
9801fae29f ignore: support compilation on wasm 
Currently the crate assumes that exactly one of `cfg(windows)` or
`cfg(unix)` is true, but this is not actually the case, for instance
when compiling for `wasm32`.

Implement the missing functions so that the crate can compile on other
platforms, even though those functions will always return an error.

PR #1327
 2019-07-24 12:55:37 -04:00
Miloš Stojanović
abdf7140d7 readme: fix broken link to Scoop bucket 
PR #1324
 2019-07-20 12:03:46 -04:00
Conrad Olega
b83e7968ef ignore/types: add Robot Framework 
PR #1322
 2019-07-14 08:12:34 -04:00
Hugo Locurcio
8ebc113847 doc: improve docs for --replace flag 
Specifically, we document shell-specific caveats related to the `--replace`
flag.

PR #1318
 2019-07-04 11:42:35 -04:00
Andrew Gallant
785c1f1766 release: globset, grep-cli, grep-printer, grep-searcher 2019-06-26 16:53:30 -04:00
Andrew Gallant
8b734cb490 deps: update everything 2019-06-26 16:51:06 -04:00
Andrew Gallant
b93762ea7a bstr: update everything to bstr 0.2 2019-06-26 16:47:33 -04:00
Andrew Gallant
34677d2622 search: a few small touchups 2019-06-18 20:23:47 -04:00
Andrew Gallant
d1389db2e3 search: better errors for preprocessor commands 
If a preprocessor command could not be started, we now show some
additional context with the error message. Previously, it showed
something like this:

  some/file: No such file or directory (os error 2)

Which is itself pretty misleading. Now it shows:

  some/file: preprocessor command could not start: '"nonexist" "some/file"': No such file or directory (os error 2)

Fixes #1302
 2019-06-16 19:02:02 -04:00
Andrew Gallant
50bcb7409e deps: update everything 2019-06-16 18:38:45 -04:00
Andrew Gallant
7b9972c308 style: fix deprecations 
Use `dyn` for trait objects and use `..=` for inclusive ranges.
 2019-06-16 18:37:51 -04:00
Hitesh Jasani
9f000c2910 ignore/types: add more nim types 
PR #1297
 2019-06-12 14:02:28 -04:00
skierpage
392682d352 doc: point regex doc link to the latest version 
The latest doc is different, e.g. adds "symmetric differences" under
https://docs.rs/regex/*/regex/#character-classes

PR #1287
 2019-06-01 08:44:55 -04:00
Andrew Gallant
7d3f794588 ignore: remove .git check in some cases 
When we know we aren't going to process gitignores, we shouldn't waste
the syscall in every directory to check for a git repo.
 2019-05-29 18:06:11 -04:00
bruce-one
290fd2a7b6 readme: mention Zstandard and Brotli 
Also alphabetise the list.

PR #1288
 2019-05-29 13:37:31 -04:00
Fabian Würfl
d1e4d28f30 readme: remove outdated statement 
Issue #10 already states that "ripgrep is now in most or all of the major
package repositories."

PR #1280
 2019-05-14 18:44:50 -04:00
Andrew Gallant
5ce2d7351d ci: use cross for musl x86_64 builds 
This is necessary because jemalloc + musl + Ubuntu 16.04 is apparently
broken.

Moreover, jemalloc doesn't support i686, so we accept the performance
regression there.

See also: https://github.com/gnzlbg/jemallocator/issues/124
 2019-04-25 11:12:14 -04:00
Andrew Gallant
9dcfd9a205 deps: bump pcre2-sys to 0.2.1 
This brings in a bug fix that no longer tries to run `git` to update the
submodule if the `git` command doesn't exist.

This is useful is more restricted build contexts where `git` isn't
installed. Such as in the docker image used for running `cross`.
 2019-04-25 11:12:14 -04:00
Andrew Gallant
36b276c6d0 printer: remove unnecessary mut 2019-04-24 17:22:27 -04:00
Andrew Gallant
03bf37ff4a alloc: use jemalloc when building with musl 
It turns out that musl's allocator is slow enough to cause a fairly
noticeable performance regression when ripgrep is built as a static
binary with musl. We fix this by using jemalloc when building with musl.

We continue to use the default system allocator in all other scenarios.
Namely, glibc's allocator doesn't noticeably regress performance compared
to jemalloc. But we could add more targets to this logic if other
system allocators (macOS, Windows) prove to be slow.

This wasn't necessary before because rustc recently stopped using jemalloc
by default.

Fixes #1268
 2019-04-24 17:21:38 -04:00
Andrew Gallant
e7829c05d3 cli: fix bug where last byte was stripped 
In an effort to strip line terminators, we assumed their existence. But
a pattern file may not end with a line terminator, so we shouldn't
unconditionally strip them.

We fix this by moving to bstr's line handling, which does this for us
automatically.
 2019-04-19 07:11:44 -04:00
Rory O’Kane
a6222939f9 readme: mention --pcre2 as long form of -P 
This is for consistency with the short and long flags given in other
bullet points. I originally assumed there was no long flag for `-P`
because none was given here.

PR #1254
 2019-04-16 21:22:48 -04:00
Rory O’Kane
6ffd434232 readme: mention --auto-hybrid-regex in advantages 
This feature solves a major reason I was skeptical of using ripgrep, so
I think it’s good to mention it in the section about why one should use
it.

I use backreferences a lot, so I had previously thought that ripgrep
would provide no speed advantage over ag, since I would always have
`-P` enabled. But when I saw `--auto-hybrid-regex` in the 11.0.0
changelog, I learned that ripgrep can use it to speed up simple queries
while still allowing me to write backreferences.

PR #1253
 2019-04-16 17:21:40 -04:00
Andrew Gallant
1f1cd9b467 pkg: update brew tap to 11.0.1 2019-04-16 13:39:56 -04:00
Andrew Gallant
973de50c9e ripgrep: release 11.0.1, take 2 2019-04-16 13:11:28 -04:00
Andrew Gallant
5f8805a496 ripgrep: release 11.0.1 2019-04-16 13:10:29 -04:00
Andrew Gallant
fdde2bcd38 deps: update regex to 1.1.6 
This brings in a fix for a regression introduced in ripgrep 11.

Fixes #1247
 2019-04-16 08:34:30 -04:00
Gerard de Melo
7b3fe6b325 doc: fix typo in FAQ 
PR #1248
 2019-04-16 08:32:30 -04:00
Max Horn
b3dd3ae203 ignore/types: add GAP 
Add support for file types used by the GAP language, a research system
computational discrete algebra, see <https://www.gap-system.org>

PR #1249
 2019-04-16 08:31:58 -04:00
Andrew Gallant
f3083e4574 readme: remove brew tap instructions 
The brew tap isn't really needed any more, since SIMD is now
automatically enabled in all binaries.
 2019-04-15 18:32:33 -04:00
Andrew Gallant
d03e30707e pkg: update brew tap to 11.0.0 2019-04-15 18:32:10 -04:00
Andrew Gallant
d7f57d9aab ripgrep: release 11.0.0 2019-04-15 18:09:40 -04:00
Andrew Gallant
1a2a24ea74 grep: release 0.2.4 2019-04-15 18:03:46 -04:00
Andrew Gallant
d66610b295 grep-cli: release 0.1.2 2019-04-15 18:02:44 -04:00
Andrew Gallant
019ae1989b grep-printer: release 0.1.2 2019-04-15 18:00:49 -04:00
Andrew Gallant
36d3f235dc grep-searcher: release 0.1.4 2019-04-15 17:59:22 -04:00
160 changed files with 5824 additions and 4255 deletions
Expand all files Collapse all files

195
.github/workflows/ci.yml vendored Normal file
View File

@@ -0,0 +1,195 @@
name: ci
on:
pull_request:
push:
branches:
- master
schedule:
- cron: '00 01 * * *'
jobs:
test:
name: test
env:
# For some builds, we use cross to test on 32-bit and big-endian
# systems.
CARGO: cargo
# When CARGO is set to CROSS, this is set to `--target matrix.target`.
TARGET_FLAGS:
# When CARGO is set to CROSS, TARGET_DIR includes matrix.target.
TARGET_DIR: ./target
# Emit backtraces on panics.
RUST_BACKTRACE: 1
runs-on: ${{ matrix.os }}
strategy:
matrix:
build:
# We test ripgrep on a pinned version of Rust, along with the moving
# targets of 'stable' and 'beta' for good measure.
- pinned
- stable
- beta
# Our release builds are generated by a nightly compiler to take
# advantage of the latest optimizations/compile time improvements. So
# we test all of them here. (We don't do mips releases, but test on
# mips for big-endian coverage.)
- nightly
- nightly-musl
- nightly-32
- nightly-mips
- nightly-arm
- macos
- win-msvc
- win-gnu
include:
- build: pinned
os: ubuntu-18.04
rust: 1.41.0
- build: stable
os: ubuntu-18.04
rust: stable
- build: beta
os: ubuntu-18.04
rust: beta
- build: nightly
os: ubuntu-18.04
rust: nightly
- build: nightly-musl
os: ubuntu-18.04
rust: nightly
target: x86_64-unknown-linux-musl
- build: nightly-32
os: ubuntu-18.04
rust: nightly
target: i686-unknown-linux-gnu
- build: nightly-mips
os: ubuntu-18.04
rust: nightly
target: mips64-unknown-linux-gnuabi64
- build: nightly-arm
os: ubuntu-18.04
rust: nightly
# For stripping release binaries:
# docker run --rm -v $PWD/target:/target:Z \
# rustembedded/cross:arm-unknown-linux-gnueabihf \
# arm-linux-gnueabihf-strip \
# /target/arm-unknown-linux-gnueabihf/debug/rg
target: arm-unknown-linux-gnueabihf
- build: macos
os: macos-latest
rust: nightly
- build: win-msvc
os: windows-2019
rust: nightly
- build: win-gnu
os: windows-2019
rust: nightly-x86_64-gnu
steps:
- name: Checkout repository
uses: actions/checkout@v1
- name: Install packages (Ubuntu)
if: matrix.os == 'ubuntu-18.04'
run: |
ci/ubuntu-install-packages
- name: Install packages (macOS)
if: matrix.os == 'macos-latest'
run: |
ci/macos-install-packages
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: ${{ matrix.rust }}
profile: minimal
override: true
- name: Use Cross
if: matrix.target != ''
run: |
# FIXME: to work around bugs in latest cross release, install master.
# See: https://github.com/rust-embedded/cross/issues/357
cargo install --git https://github.com/rust-embedded/cross
echo "::set-env name=CARGO::cross"
echo "::set-env name=TARGET_FLAGS::--target ${{ matrix.target }}"
echo "::set-env name=TARGET_DIR::./target/${{ matrix.target }}"
- name: Show command used for Cargo
run: |
echo "cargo command is: ${{ env.CARGO }}"
echo "target flag is: ${{ env.TARGET_FLAGS }}"
- name: Build ripgrep and all crates
run: ${{ env.CARGO }} build --verbose --all ${{ env.TARGET_FLAGS }}
- name: Build ripgrep with PCRE2
run: ${{ env.CARGO }} build --verbose --all --features pcre2 ${{ env.TARGET_FLAGS }}
# This is useful for debugging problems when the expected build artifacts
# (like shell completions and man pages) aren't generated.
- name: Show build.rs stderr
shell: bash
run: |
set +x
stderr="$(find "${{ env.TARGET_DIR }}/debug" -name stderr -print0 | xargs -0 ls -t | head -n1)"
if [ -s "$stderr" ]; then
echo "===== $stderr ===== "
cat "$stderr"
echo "====="
fi
set -x
- name: Run tests with PCRE2 (sans cross)
if: matrix.target == ''
run: ${{ env.CARGO }} test --verbose --all --features pcre2 ${{ env.TARGET_FLAGS }}
- name: Run tests without PCRE2 (with cross)
# These tests should actually work, but they almost double the runtime.
# Every integration test spins up qemu to run 'rg', and when PCRE2 is
# enabled, every integration test is run twice: one with the default
# regex engine and once with PCRE2.
if: matrix.target != ''
run: ${{ env.CARGO }} test --verbose --all ${{ env.TARGET_FLAGS }}
- name: Test for existence of build artifacts (Windows)
if: matrix.os == 'windows-2019'
shell: bash
run: |
outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
ls "$outdir/_rg.ps1" && file "$outdir/_rg.ps1"
- name: Test for existence of build artifacts (Unix)
if: matrix.os != 'windows-2019'
shell: bash
run: |
outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
for f in rg.bash rg.fish rg.1; do
# We could use file -E here, but it isn't supported on macOS.
ls "$outdir/$f" && file "$outdir/$f"
done
- name: Test zsh shell completions (Unix, sans cross)
# We could test this when using Cross, but we'd have to execute the
# 'rg' binary (done in test-complete) with qemu, which is a pain and
# doesn't really gain us much. If shell completion works in one place,
# it probably works everywhere.
if: matrix.target == '' && matrix.os != 'windows-2019'
shell: bash
run: ci/test-complete
rustfmt:
name: rustfmt
runs-on: ubuntu-18.04
steps:
- name: Checkout repository
uses: actions/checkout@v1
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
override: true
profile: minimal
components: rustfmt
- name: Check formatting
run: |
cargo fmt --all -- --check

211
.github/workflows/release.yml vendored Normal file
View File

@@ -0,0 +1,211 @@
# The way this works is a little weird. But basically, the create-release job
# runs purely to initialize the GitHub release itself. Once done, the upload
# URL of the release is saved as an artifact.
#
# The build-release job runs only once create-release is finished. It gets
# the release upload URL by downloading the corresponding artifact (which was
# uploaded by create-release). It then builds the release executables for each
# supported platform and attaches them as release assets to the previously
# created release.
#
# The key here is that we create the release only once.
name: release
on:
push:
# Enable when testing release infrastructure on a branch.
# branches:
# - ag/release
tags:
- '[0-9]+.[0-9]+.[0-9]+'
jobs:
create-release:
name: create-release
runs-on: ubuntu-latest
env:
# Set to force version number, e.g., when no tag exists.
# RG_VERSION: TEST-0.0.0
steps:
- name: Create artifacts directory
run: mkdir artifacts
- name: Get the release version from the tag
if: env.RG_VERSION == ''
run: |
# Apparently, this is the right way to get a tag name. Really?
#
# See: https://github.community/t5/GitHub-Actions/How-to-get-just-the-tag-name/m-p/32167/highlight/true#M1027
echo "::set-env name=RG_VERSION::${GITHUB_REF#refs/tags/}"
echo "version is: ${{ env.RG_VERSION }}"
- name: Create GitHub release
id: release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ env.RG_VERSION }}
release_name: ripgrep ${{ env.RG_VERSION }}
- name: Save release upload URL to artifact
run: echo "${{ steps.release.outputs.upload_url }}" > artifacts/release-upload-url
- name: Save version number to artifact
run: echo "${{ env.RG_VERSION }}" > artifacts/release-version
- name: Upload artifacts
uses: actions/upload-artifact@v1
with:
name: artifacts
path: artifacts
build-release:
name: build-release
needs: ['create-release']
runs-on: ${{ matrix.os }}
env:
# For some builds, we use cross to test on 32-bit and big-endian
# systems.
CARGO: cargo
# When CARGO is set to CROSS, this is set to `--target matrix.target`.
TARGET_FLAGS:
# When CARGO is set to CROSS, TARGET_DIR includes matrix.target.
TARGET_DIR: ./target
# Emit backtraces on panics.
RUST_BACKTRACE: 1
# Build static releases with PCRE2.
PCRE2_SYS_STATIC: 1
strategy:
matrix:
build: [linux, linux-arm, macos, win-msvc, win-gnu, win32-msvc]
include:
- build: linux
os: ubuntu-18.04
rust: nightly
target: x86_64-unknown-linux-musl
- build: linux-arm
os: ubuntu-18.04
rust: nightly
target: arm-unknown-linux-gnueabihf
- build: macos
os: macos-latest
rust: nightly
target: x86_64-apple-darwin
- build: win-msvc
os: windows-2019
rust: nightly
target: x86_64-pc-windows-msvc
- build: win-gnu
os: windows-2019
rust: nightly-x86_64-gnu
target: x86_64-pc-windows-gnu
- build: win32-msvc
os: windows-2019
rust: nightly
target: i686-pc-windows-msvc
steps:
- name: Checkout repository
uses: actions/checkout@v1
with:
fetch-depth: 1
- name: Install packages (Ubuntu)
if: matrix.os == 'ubuntu-18.04'
run: |
ci/ubuntu-install-packages
- name: Install packages (macOS)
if: matrix.os == 'macos-latest'
run: |
ci/macos-install-packages
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: ${{ matrix.rust }}
profile: minimal
override: true
target: ${{ matrix.target }}
- name: Use Cross
# if: matrix.os != 'windows-2019'
run: |
# FIXME: to work around bugs in latest cross release, install master.
# See: https://github.com/rust-embedded/cross/issues/357
cargo install --git https://github.com/rust-embedded/cross
echo "::set-env name=CARGO::cross"
echo "::set-env name=TARGET_FLAGS::--target ${{ matrix.target }}"
echo "::set-env name=TARGET_DIR::./target/${{ matrix.target }}"
- name: Show command used for Cargo
run: |
echo "cargo command is: ${{ env.CARGO }}"
echo "target flag is: ${{ env.TARGET_FLAGS }}"
echo "target dir is: ${{ env.TARGET_DIR }}"
- name: Get release download URL
uses: actions/download-artifact@v1
with:
name: artifacts
path: artifacts
- name: Set release upload URL and release version
shell: bash
run: |
release_upload_url="$(cat artifacts/release-upload-url)"
echo "::set-env name=RELEASE_UPLOAD_URL::$release_upload_url"
echo "release upload url: $RELEASE_UPLOAD_URL"
release_version="$(cat artifacts/release-version)"
echo "::set-env name=RELEASE_VERSION::$release_version"
echo "release version: $RELEASE_VERSION"
- name: Build release binary
run: ${{ env.CARGO }} build --verbose --release --features pcre2 ${{ env.TARGET_FLAGS }}
- name: Strip release binary (linux and macos)
if: matrix.build == 'linux' || matrix.build == 'macos'
run: strip "target/${{ matrix.target }}/release/rg"
- name: Strip release binary (arm)
if: matrix.build == 'linux-arm'
run: |
docker run --rm -v \
"$PWD/target:/target:Z" \
rustembedded/cross:arm-unknown-linux-gnueabihf \
arm-linux-gnueabihf-strip \
/target/arm-unknown-linux-gnueabihf/release/rg
- name: Build archive
shell: bash
run: |
outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
staging="ripgrep-${{ env.RELEASE_VERSION }}-${{ matrix.target }}"
mkdir -p "$staging"/{complete,doc}
cp {README.md,COPYING,UNLICENSE,LICENSE-MIT} "$staging/"
cp {CHANGELOG.md,FAQ.md,GUIDE.md} "$staging/doc/"
cp "$outdir"/{rg.bash,rg.fish,_rg.ps1} "$staging/complete/"
cp complete/_rg "$staging/complete/"
if [ "${{ matrix.os }}" = "windows-2019" ]; then
cp "target/${{ matrix.target }}/release/rg.exe" "$staging/"
7z a "$staging.zip" "$staging"
echo "::set-env name=ASSET::$staging.zip"
else
# The man page is only generated on Unix systems. ¯\_(ツ)_/¯
cp "$outdir"/rg.1 "$staging/doc/"
cp "target/${{ matrix.target }}/release/rg" "$staging/"
tar czf "$staging.tar.gz" "$staging"
echo "::set-env name=ASSET::$staging.tar.gz"
fi
- name: Upload release archive
uses: actions/upload-release-asset@v1.0.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ env.RELEASE_UPLOAD_URL }}
asset_path: ${{ env.ASSET }}
asset_name: ${{ env.ASSET }}
asset_content_type: application/octet-stream

110
.travis.yml
View File

@@ -1,110 +0,0 @@
language: rust
dist: xenial
env:
global:
- PROJECT_NAME: ripgrep
- RUST_BACKTRACE: full
addons:
apt:
packages:
# For generating man page.
- libxslt1-dev
- asciidoc
- docbook-xsl
- xsltproc
- libxml2-utils
# Needed for completion-function test.
- zsh
# Needed for testing decompression search.
- xz-utils
- liblz4-tool
# For building MUSL static builds on Linux.
- musl-tools
matrix:
fast_finish: true
include:
# Nightly channel.
# All *nix releases are done on the nightly channel to take advantage
# of the regex library's multiple pattern SIMD search.
- os: linux
rust: nightly
env: TARGET=i686-unknown-linux-musl
- os: linux
rust: nightly
env: TARGET=x86_64-unknown-linux-musl
- os: osx
rust: nightly
# XML_CATALOG_FILES is apparently necessary for asciidoc on macOS.
env: TARGET=x86_64-apple-darwin XML_CATALOG_FILES=/usr/local/etc/xml/catalog
- os: linux
rust: nightly
env: TARGET=arm-unknown-linux-gnueabihf GCC_VERSION=4.8
addons:
apt:
packages:
- gcc-4.8-arm-linux-gnueabihf
- binutils-arm-linux-gnueabihf
- libc6-armhf-cross
- libc6-dev-armhf-cross
# For generating man page.
- libxslt1-dev
- asciidoc
- docbook-xsl
- xsltproc
- libxml2-utils
# Beta channel. We enable these to make sure there are no regressions in
# Rust beta releases.
- os: linux
rust: beta
env: TARGET=x86_64-unknown-linux-musl
- os: linux
rust: beta
env: TARGET=x86_64-unknown-linux-gnu
# Minimum Rust supported channel. We enable these to make sure ripgrep
# continues to work on the advertised minimum Rust version.
- os: linux
rust: 1.34.0
env: TARGET=x86_64-unknown-linux-gnu
- os: linux
rust: 1.34.0
env: TARGET=x86_64-unknown-linux-musl
- os: linux
rust: 1.34.0
env: TARGET=arm-unknown-linux-gnueabihf GCC_VERSION=4.8
addons:
apt:
packages:
- gcc-4.8-arm-linux-gnueabihf
- binutils-arm-linux-gnueabihf
- libc6-armhf-cross
- libc6-dev-armhf-cross
# For generating man page.
- libxslt1-dev
- asciidoc
- docbook-xsl
- xsltproc
- libxml2-utils
install: ci/install.sh
script: ci/script.sh
before_deploy: ci/before_deploy.sh
deploy:
provider: releases
file_glob: true
file: deployment/${PROJECT_NAME}-${TRAVIS_TAG}-${TARGET}.tar.gz
skip_cleanup: true
on:
condition: $TRAVIS_RUST_VERSION = nightly
branch: master # i guess we do need this after all?
tags: true
api_key:
secure: "IbSnsbGkxSydR/sozOf1/SRvHplzwRUHzcTjM7BKnr7GccL86gRPUrsrvD103KjQUGWIc1TnK1YTq5M0Onswg/ORDjqa1JEJPkPdPnVh9ipbF7M2De/7IlB4X4qXLKoApn8+bx2x/mfYXu4G+G1/2QdbaKK2yfXZKyjz0YFx+6CNrVCT2Nk8q7aHvOOzAL58vsG8iPDpupuhxlMDDn/UhyOWVInmPPQ0iJR1ZUJN8xJwXvKvBbfp3AhaBiAzkhXHNLgBR8QC5noWWMXnuVDMY3k4f3ic0V+p/qGUCN/nhptuceLxKFicMCYObSZeUzE5RAI0/OBW7l3z2iCoc+TbAnn+JrX/ObJCfzgAOXAU3tLaBFMiqQPGFKjKg1ltSYXomOFP/F7zALjpvFp4lYTBajRR+O3dqaxA9UQuRjw27vOeUpMcga4ZzL4VXFHzrxZKBHN//XIGjYAVhJ1NSSeGpeJV5/+jYzzWKfwSagRxQyVCzMooYFFXzn8Yxdm3PJlmp3GaAogNkdB9qKcrEvRINCelalzALPi0hD/HUDi8DD2PNTCLLMo6VSYtvc685Zbe+KgNzDV1YyTrRCUW6JotrS0r2ULLwnsh40hSB//nNv3XmwNmC/CmW5QAnIGj8cBMF4S2t6ohADIndojdAfNiptmaZOIT6owK7bWMgPMyopo="
branches:
only:
# Pushes and PR to the master branch
- master
# Ruby regex to match tags. Required, or travis won't trigger deploys when
# a new tag is pushed.
- /^\d+\.\d+\.\d+.*$/
notifications:
email:
on_success: never

137
CHANGELOG.md
View File

@@ -1,5 +1,132 @@
11.0.0 (TBD) 12.0.0 (2020-03-15)
============ ===================
ripgrep 12 is a new major version release of ripgrep that contains many bug
fixes, several important performance improvements and a few minor new features.
In a near future release, I am hoping to add an
[indexing feature](https://github.com/BurntSushi/ripgrep/issues/1497)
to ripgrep, which will dramatically speed up searching by building an index.
Feedback would very much be appreciated, especially on the user experience
which will be difficult to get right.
This release has no known breaking changes.
Deprecations:
* The `--no-pcre2-unicode` flag is deprecated. Instead, use the `--no-unicode`
flag, which applies to both the default regex engine and PCRE2. For now,
`--no-pcre2-unicode` and `--pcre2-unicode` are aliases to `--no-unicode`
and `--unicode`, respectively. The `--[no-]pcre2-unicode` flags may be
removed in a future release.
* The `--auto-hybrid-regex` flag is deprecated. Instead, use the new `--engine`
flag with the `auto` value.
Performance improvements:
* [PERF #1087](https://github.com/BurntSushi/ripgrep/pull/1087):
ripgrep is smarter when detected literals are whitespace.
* [PERF #1381](https://github.com/BurntSushi/ripgrep/pull/1381):
Directory traversal is sped up with speculative ignore-file existence checks.
* [PERF cd8ec38a](https://github.com/BurntSushi/ripgrep/commit/cd8ec38a):
Improve inner literal detection to cover more cases more effectively.
e.g., ` +Sherlock Holmes +` now has ` Sherlock Holmes ` extracted instead
of ` `.
* [PERF 6a0e0147](https://github.com/BurntSushi/ripgrep/commit/6a0e0147):
Improve literal detection when the `-w/--word-regexp` flag is used.
* [PERF ad97e9c9](https://github.com/BurntSushi/ripgrep/commit/ad97e9c9):
Improve overall performance of the `-w/--word-regexp` flag.
Feature enhancements:
* Added or improved file type filtering for erb, diff, Gradle, HAML, Org,
Postscript, Skim, Slim, Slime, RPM Spec files, Typoscript, xml.
* [FEATURE #1370](https://github.com/BurntSushi/ripgrep/pull/1370):
Add `--include-zero` flag that shows files searched without matches.
* [FEATURE #1390](https://github.com/BurntSushi/ripgrep/pull/1390):
Add `--no-context-separator` flag that always hides context separators.
* [FEATURE #1414](https://github.com/BurntSushi/ripgrep/pull/1414):
Add `--no-require-git` flag to allow ripgrep to respect gitignores anywhere.
* [FEATURE #1420](https://github.com/BurntSushi/ripgrep/pull/1420):
Add `--no-ignore-exclude` to disregard rules in `.git/info/exclude` files.
* [FEATURE #1466](https://github.com/BurntSushi/ripgrep/pull/1466):
Add `--no-ignore-files` flag to disable all `--ignore-file` flags.
* [FEATURE #1488](https://github.com/BurntSushi/ripgrep/pull/1488):
Add '--engine' flag for easier switching between regex engines.
* [FEATURE 75cbe88f](https://github.com/BurntSushi/ripgrep/commit/75cbe88f):
Add `--no-unicode` flag. This works on all supported regex engines.
Bug fixes:
* [BUG #1291](https://github.com/BurntSushi/ripgrep/issues/1291):
ripgrep now works in non-existent directories.
* [BUG #1319](https://github.com/BurntSushi/ripgrep/issues/1319):
Fix match bug due to errant literal detection.
* [**BUG #1335**](https://github.com/BurntSushi/ripgrep/issues/1335):
Fixes a performance bug when searching plain text files with very long lines.
This was a serious performance regression in some cases.
* [BUG #1344](https://github.com/BurntSushi/ripgrep/issues/1344):
Document usage of `--type all`.
* [BUG #1389](https://github.com/BurntSushi/ripgrep/issues/1389):
Fixes a bug where ripgrep would panic when searching a symlinked directory.
* [BUG #1439](https://github.com/BurntSushi/ripgrep/issues/1439):
Improve documentation for ripgrep's automatic stdin detection.
* [BUG #1441](https://github.com/BurntSushi/ripgrep/issues/1441):
Remove CPU features from man page.
* [BUG #1442](https://github.com/BurntSushi/ripgrep/issues/1442),
[BUG #1478](https://github.com/BurntSushi/ripgrep/issues/1478):
Improve documentation of the `-g/--glob` flag.
* [BUG #1445](https://github.com/BurntSushi/ripgrep/issues/1445):
ripgrep now respects ignore rules from .git/info/exclude in worktrees.
* [BUG #1485](https://github.com/BurntSushi/ripgrep/issues/1485):
Fish shell completions from the release Debian package are now installed to
`/usr/share/fish/vendor_completions.d/rg.fish`.
11.0.2 (2019-08-01)
===================
ripgrep 11.0.2 is a new patch release that fixes a few bugs, including a
performance regression and a matching bug when using the `-F/--fixed-strings`
flag.
Feature enhancements:
* [FEATURE #1293](https://github.com/BurntSushi/ripgrep/issues/1293):
Added `--glob-case-insensitive` flag that makes `--glob` behave as `--iglob`.
Bug fixes:
* [BUG #1246](https://github.com/BurntSushi/ripgrep/issues/1246):
Add translations to README, starting with an unofficial Chinese translation.
* [BUG #1259](https://github.com/BurntSushi/ripgrep/issues/1259):
Fix bug where the last byte of a `-f file` was stripped if it wasn't a `\n`.
* [BUG #1261](https://github.com/BurntSushi/ripgrep/issues/1261):
Document that no error is reported when searching for `\n` with `-P/--pcre2`.
* [BUG #1284](https://github.com/BurntSushi/ripgrep/issues/1284):
Mention `.ignore` and `.rgignore` more prominently in the README.
* [BUG #1292](https://github.com/BurntSushi/ripgrep/issues/1292):
Fix bug where `--with-filename` was sometimes enabled incorrectly.
* [BUG #1268](https://github.com/BurntSushi/ripgrep/issues/1268):
Fix major performance regression in GitHub `x86_64-linux` binary release.
* [BUG #1302](https://github.com/BurntSushi/ripgrep/issues/1302):
Show better error messages when a non-existent preprocessor command is given.
* [BUG #1334](https://github.com/BurntSushi/ripgrep/issues/1334):
Fix match regression with `-F` flag when patterns contain meta characters.
11.0.1 (2019-04-16)
===================
ripgrep 11.0.1 is a new patch release that fixes a search regression introduced
in the previous 11.0.0 release. In particular, ripgrep can enter an infinite
loop for some search patterns when searching invalid UTF-8.
Bug fixes:
* [BUG #1247](https://github.com/BurntSushi/ripgrep/issues/1247):
Fix search bug that can cause ripgrep to enter an infinite loop.
11.0.0 (2019-04-15)
===================
ripgrep 11 is a new major version release of ripgrep that contains many bug ripgrep 11 is a new major version release of ripgrep that contains many bug
fixes, some performance improvements and a few feature enhancements. Notably, fixes, some performance improvements and a few feature enhancements. Notably,
ripgrep's user experience for binary file filtering has been improved. See the ripgrep's user experience for binary file filtering has been improved. See the
@@ -32,9 +159,9 @@ This release increases the **minimum supported Rust version** from 1.28.0 to
terminal. That is, `rg -uuu foo` should now be equivalent to `grep -r foo`. terminal. That is, `rg -uuu foo` should now be equivalent to `grep -r foo`.
* The `avx-accel` feature of ripgrep has been removed since it is no longer * The `avx-accel` feature of ripgrep has been removed since it is no longer
necessary. All uses of AVX in ripgrep are now enabled automatically via necessary. All uses of AVX in ripgrep are now enabled automatically via
runtime CPU feature detection. The `simd-accel` feature does remain runtime CPU feature detection. The `simd-accel` feature does remain available
available, however, it does increase compilation times substantially at the (only for enabling SIMD for transcoding), however, it does increase
moment. compilation times substantially at the moment.
Performance improvements: Performance improvements:

685
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

46
Cargo.toml
View File

@@ -1,11 +1,11 @@
[package] [package]
name = "ripgrep" name = "ripgrep"
version = "0.10.0" #:version version = "12.0.0" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"] authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """ description = """
ripgrep is a line-oriented search tool that recursively searches your current ripgrep is a line-oriented search tool that recursively searches your current
directory for a regex pattern while respecting your gitignore rules. ripgrep directory for a regex pattern while respecting your gitignore rules. ripgrep
has first class support on Windows, macOS and Linux has first class support on Windows, macOS and Linux.
""" """
documentation = "https://github.com/BurntSushi/ripgrep" documentation = "https://github.com/BurntSushi/ripgrep"
homepage = "https://github.com/BurntSushi/ripgrep" homepage = "https://github.com/BurntSushi/ripgrep"
@@ -25,7 +25,7 @@ appveyor = { repository = "BurntSushi/ripgrep" }
[[bin]] [[bin]]
bench = false bench = false
path = "src/main.rs" path = "crates/core/main.rs"
name = "rg" name = "rg"
[[test]] [[test]]
@@ -34,44 +34,48 @@ path = "tests/tests.rs"
[workspace] [workspace]
members = [ members = [
"globset", "crates/globset",
"grep", "crates/grep",
"grep-cli", "crates/cli",
"grep-matcher", "crates/matcher",
"grep-pcre2", "crates/pcre2",
"grep-printer", "crates/printer",
"grep-regex", "crates/regex",
"grep-searcher", "crates/searcher",
"ignore", "crates/ignore",
] ]
[dependencies] [dependencies]
bstr = "0.1.2" bstr = "0.2.12"
grep = { version = "0.2.3", path = "grep" } grep = { version = "0.2.5", path = "crates/grep" }
ignore = { version = "0.4.7", path = "ignore" } ignore = { version = "0.4.12", path = "crates/ignore" }
lazy_static = "1.1.0" lazy_static = "1.1.0"
log = "0.4.5" log = "0.4.5"
num_cpus = "1.8.0" num_cpus = "1.8.0"
regex = "1.0.5" regex = "1.3.5"
serde_json = "1.0.23" serde_json = "1.0.23"
termcolor = "1.0.3" termcolor = "1.1.0"
[dependencies.clap] [dependencies.clap]
version = "2.32.0" version = "2.33.0"
default-features = false default-features = false
features = ["suggestions"] features = ["suggestions"]
[target.'cfg(all(target_env = "musl", target_pointer_width = "64"))'.dependencies.jemallocator]
version = "0.3.0"
[build-dependencies] [build-dependencies]
lazy_static = "1.1.0" lazy_static = "1.1.0"
[build-dependencies.clap] [build-dependencies.clap]
version = "2.32.0" version = "2.33.0"
default-features = false default-features = false
features = ["suggestions"] features = ["suggestions"]
[dev-dependencies] [dev-dependencies]
serde = "1.0.77" serde = "1.0.77"
serde_derive = "1.0.77" serde_derive = "1.0.77"
walkdir = "2"
[features] [features]
simd-accel = ["grep/simd-accel"] simd-accel = ["grep/simd-accel"]
@@ -94,11 +98,11 @@ assets = [
# The man page is automatically generated by ripgrep's build process, so # The man page is automatically generated by ripgrep's build process, so
# this file isn't actually commited. Instead, to create a dpkg, either # this file isn't actually commited. Instead, to create a dpkg, either
# create a deployment/deb directory and copy the man page to it, or use the # create a deployment/deb directory and copy the man page to it, or use the
# 'ci/build_deb.sh' script. # 'ci/build-deb' script.
["deployment/deb/rg.1", "usr/share/man/man1/rg.1", "644"], ["deployment/deb/rg.1", "usr/share/man/man1/rg.1", "644"],
# Similarly for shell completions. # Similarly for shell completions.
["deployment/deb/rg.bash", "usr/share/bash-completion/completions/rg", "644"], ["deployment/deb/rg.bash", "usr/share/bash-completion/completions/rg", "644"],
["deployment/deb/rg.fish", "usr/share/fish/completions/rg.fish", "644"], ["deployment/deb/rg.fish", "usr/share/fish/vendor_completions.d/rg.fish", "644"],
["deployment/deb/_rg", "usr/share/zsh/vendor-completions/", "644"], ["deployment/deb/_rg", "usr/share/zsh/vendor-completions/", "644"],
] ]
extended-description = """\ extended-description = """\

11
Cross.toml Normal file
View File

@@ -0,0 +1,11 @@
[target.x86_64-unknown-linux-musl]
image = "burntsushi/cross:x86_64-unknown-linux-musl"
[target.i686-unknown-linux-gnu]
image = "burntsushi/cross:i686-unknown-linux-gnu"
[target.mips64-unknown-linux-gnuabi64]
image = "burntsushi/cross:mips64-unknown-linux-gnuabi64"
[target.arm-unknown-linux-gnueabihf]
image = "burntsushi/cross:arm-unknown-linux-gnueabihf"

29
FAQ.md
View File

@@ -25,6 +25,7 @@
* [How is ripgrep licensed?](#license) * [How is ripgrep licensed?](#license)
* [Can ripgrep replace grep?](#posix4ever) * [Can ripgrep replace grep?](#posix4ever)
* [What does the "rip" in ripgrep mean?](#intentcountsforsomething) * [What does the "rip" in ripgrep mean?](#intentcountsforsomething)
* [How can I donate to ripgrep or its maintainers?](#donations)
<h3 name="config"> <h3 name="config">
@@ -50,9 +51,9 @@ ripgrep is a project whose contributors are volunteers. A release schedule
adds undue stress to said volunteers. Therefore, releases are made on a best adds undue stress to said volunteers. Therefore, releases are made on a best
effort basis and no dates **will ever be given**. effort basis and no dates **will ever be given**.
One exception to this is high impact bugs. If a ripgrep release contains a An exception to this _can be_ high impact bugs. If a ripgrep release contains
significant regression, then there will generally be a strong push to get a a significant regression, then there will generally be a strong push to get a
patch release out with a fix. patch release out with a fix. However, no promises are made.
<h3 name="manpage"> <h3 name="manpage">
@@ -934,8 +935,8 @@ Here are some cases where you might *not* want to use ripgrep. The same caveats
for the previous section apply. for the previous section apply.
* Are you writing portable shell scripts intended to work in a variety of * Are you writing portable shell scripts intended to work in a variety of
environments? Great, probably not a good idea to use ripgrep! ripgrep is has environments? Great, probably not a good idea to use ripgrep! ripgrep has
nowhere near the ubquity of grep, so if you do use ripgrep, you might need nowhere near the ubiquity of grep, so if you do use ripgrep, you might need
to futz with the installation process more than you would with grep. to futz with the installation process more than you would with grep.
* Do you care about POSIX compatibility? If so, then you can't use ripgrep * Do you care about POSIX compatibility? If so, then you can't use ripgrep
because it never was, isn't and never will be POSIX compatible. because it never was, isn't and never will be POSIX compatible.
@@ -981,3 +982,21 @@ grep](#posix4ever),
ripgrep is neither actually a "grep killer" nor was it ever intended to be. It ripgrep is neither actually a "grep killer" nor was it ever intended to be. It
certainly does eat into some of its use cases, but that's nothing that other certainly does eat into some of its use cases, but that's nothing that other
tools like ack or The Silver Searcher weren't already doing. tools like ack or The Silver Searcher weren't already doing.
<h3 name="donations">
How can I donate to ripgrep or its maintainers?
</h3>
As of now, you can't. While I believe the various efforts that are being
undertaken to help fund FOSS are extremely important, they aren't a good fit
for me. ripgrep is and I hope will remain a project of love that I develop in
my free time. As such, involving money---even in the form of donations given
without expectations---would severely change that dynamic for me personally.
Instead, I'd recommend donating to something else that is doing work that you
find meaningful. If you would like suggestions, then my favorites are:
* [The Internet Archive](https://archive.org/donate/)
* [Rails Girls](https://railsgirlssummerofcode.org/campaign/)
* [Wikipedia](https://wikimediafoundation.org/support/)

21
GUIDE.md
View File

@@ -110,7 +110,7 @@ colors, you'll notice that `faster` will be highlighted instead of just the
It is beyond the scope of this guide to provide a full tutorial on regular It is beyond the scope of this guide to provide a full tutorial on regular
expressions, but ripgrep's specific syntax is documented here: expressions, but ripgrep's specific syntax is documented here:
https://docs.rs/regex/0.2.5/regex/#syntax https://docs.rs/regex/*/regex/#syntax
### Recursive search ### Recursive search
@@ -411,6 +411,21 @@ alias rg="rg --type-add 'web:*.{html,css,js}'"
or add `--type-add=web:*.{html,css,js}` to your ripgrep configuration file. or add `--type-add=web:*.{html,css,js}` to your ripgrep configuration file.
([Configuration files](#configuration-file) are covered in more detail later.) ([Configuration files](#configuration-file) are covered in more detail later.)
#### The special `all` file type
A special option supported by the `--type` flag is `all`. `--type all` looks
for a match in any of the supported file types listed by `--type-list`,
including those added on the command line using `--type-add`. It's equivalent
to the command `rg --type agda --type asciidoc --type asm ...`, where `...`
stands for a list of `--type` flags for the rest of the types in `--type-list`.
As an example, let's suppose you have a shell script in your current directory,
`my-shell-script`, which includes a shell library, `my-shell-library.bash`.
Both `rg --type sh` and `rg --type all` would only search for matches in
`my-shell-library.bash`, not `my-shell-script`, because the globs matched
by the `sh` file type don't include files without an extension. On the
other hand, `rg --type-not all` would search `my-shell-script` but not
`my-shell-library.bash`.
### Replacements ### Replacements
@@ -716,8 +731,8 @@ binary files:
**only applies to files searched by ripgrep as a result of recursive **only applies to files searched by ripgrep as a result of recursive
directory traversal**, which is consistent with ripgrep's other automatic directory traversal**, which is consistent with ripgrep's other automatic
filtering. For example, `rg foo .file` will search `.file` even though it filtering. For example, `rg foo .file` will search `.file` even though it
is hidden. Similarly, `rg foo binary-file` search `binary-file` in "binary" is hidden. Similarly, `rg foo binary-file` will search `binary-file` in
mode automatically. "binary" mode automatically.
2. Binary mode is similar to the default mode, except it will not always 2. Binary mode is similar to the default mode, except it will not always
stop searching after it sees a `NUL` byte. Namely, in this mode, ripgrep stop searching after it sees a `NUL` byte. Namely, in this mode, ripgrep
will continue searching a file that is known to be binary until the first will continue searching a file that is known to be binary until the first

148
README.md
View File

@@ -8,12 +8,11 @@ available for [every release](https://github.com/BurntSushi/ripgrep/releases).
ripgrep is similar to other popular search tools like The Silver Searcher, ack ripgrep is similar to other popular search tools like The Silver Searcher, ack
and grep. and grep.
[![Linux build status](https://travis-ci.org/BurntSushi/ripgrep.svg)](https://travis-ci.org/BurntSushi/ripgrep) [![Build status](https://github.com/BurntSushi/ripgrep/workflows/ci/badge.svg)](https://github.com/BurntSushi/ripgrep/actions)
[![Windows build status](https://ci.appveyor.com/api/projects/status/github/BurntSushi/ripgrep?svg=true)](https://ci.appveyor.com/project/BurntSushi/ripgrep)
[![Crates.io](https://img.shields.io/crates/v/ripgrep.svg)](https://crates.io/crates/ripgrep) [![Crates.io](https://img.shields.io/crates/v/ripgrep.svg)](https://crates.io/crates/ripgrep)
[![Packaging status](https://repology.org/badge/tiny-repos/ripgrep.svg)](https://repology.org/project/ripgrep/badges) [![Packaging status](https://repology.org/badge/tiny-repos/ripgrep.svg)](https://repology.org/project/ripgrep/badges)
Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org). Dual-licensed under MIT or the [UNLICENSE](https://unlicense.org).
### CHANGELOG ### CHANGELOG
@@ -29,60 +28,60 @@ Please see the [CHANGELOG](CHANGELOG.md) for a release history.
* [Configuration files](GUIDE.md#configuration-file) * [Configuration files](GUIDE.md#configuration-file)
* [Shell completions](FAQ.md#complete) * [Shell completions](FAQ.md#complete)
* [Building](#building) * [Building](#building)
* [Translations](#translations)
### Screenshot of search results ### Screenshot of search results
[![A screenshot of a sample search with ripgrep](http://burntsushi.net/stuff/ripgrep1.png)](http://burntsushi.net/stuff/ripgrep1.png) [![A screenshot of a sample search with ripgrep](https://burntsushi.net/stuff/ripgrep1.png)](https://burntsushi.net/stuff/ripgrep1.png)
### Quick examples comparing tools ### Quick examples comparing tools
This example searches the entire Linux kernel source tree (after running This example searches the entire
`make defconfig && make -j8`) for `[A-Z]+_SUSPEND`, where all matches must be [Linux kernel source tree](https://github.com/BurntSushi/linux)
words. Timings were collected on a system with an Intel i7-6900K 3.2 GHz, and (after running `make defconfig && make -j8`) for `[A-Z]+_SUSPEND`, where
ripgrep was compiled with SIMD enabled. all matches must be words. Timings were collected on a system with an Intel
i7-6900K 3.2 GHz.
Please remember that a single benchmark is never enough! See my Please remember that a single benchmark is never enough! See my
[blog post on ripgrep](http://blog.burntsushi.net/ripgrep/) [blog post on ripgrep](https://blog.burntsushi.net/ripgrep/)
for a very detailed comparison with more benchmarks and analysis. for a very detailed comparison with more benchmarks and analysis.
| Tool | Command | Line count | Time | | Tool | Command | Line count | Time |
| ---- | ------- | ---------- | ---- | | ---- | ------- | ---------- | ---- |
| ripgrep (Unicode) | `rg -n -w '[A-Z]+_SUSPEND'` | 450 | **0.106s** | | ripgrep (Unicode) | `rg -n -w '[A-Z]+_SUSPEND'` | 452 | **0.136s** |
| [git grep](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=C git grep -E -n -w '[A-Z]+_SUSPEND'` | 450 | 0.553s | | [git grep](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `git grep -P -n -w '[A-Z]+_SUSPEND'` | 452 | 0.348s |
| [The Silver Searcher](https://github.com/ggreer/the_silver_searcher) | `ag -w '[A-Z]+_SUSPEND'` | 450 | 0.589s | | [ugrep (Unicode)](https://github.com/Genivia/ugrep) | `ugrep -r --ignore-files --no-hidden -I -w '[A-Z]+_SUSPEND'` | 452 | 0.506s |
| [git grep (Unicode)](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=en_US.UTF-8 git grep -E -n -w '[A-Z]+_SUSPEND'` | 450 | 2.266s | | [git grep](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=C git grep -E -n -w '[A-Z]+_SUSPEND'` | 452 | 1.150s |
| [sift](https://github.com/svent/sift) | `sift --git -n -w '[A-Z]+_SUSPEND'` | 450 | 3.505s | | [The Silver Searcher](https://github.com/ggreer/the_silver_searcher) | `ag -w '[A-Z]+_SUSPEND'` | 452 | 0.654s |
| [ack](https://github.com/petdance/ack2) | `ack -w '[A-Z]+_SUSPEND'` | 1878 | 6.823s | | [ack](https://github.com/beyondgrep/ack3) | `ack -w '[A-Z]+_SUSPEND'` | 452 | 4.054s |
| [The Platinum Searcher](https://github.com/monochromegane/the_platinum_searcher) | `pt -w -e '[A-Z]+_SUSPEND'` | 450 | 14.208s | | [git grep (Unicode)](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=en_US.UTF-8 git grep -E -n -w '[A-Z]+_SUSPEND'` | 452 | 4.205s |
(Yes, `ack` [has](https://github.com/petdance/ack2/issues/445) a Here's another benchmark on the same corpus as above that disregards gitignore
[bug](https://github.com/petdance/ack2/issues/14).) files and searches with a whitelist instead. The corpus is the same as in the
previous benchmark, and the flags passed to each command ensure that they are
Here's another benchmark that disregards gitignore files and searches with a doing equivalent work:
whitelist instead. The corpus is the same as in the previous benchmark, and the
flags passed to each command ensure that they are doing equivalent work:
| Tool | Command | Line count | Time | | Tool | Command | Line count | Time |
| ---- | ------- | ---------- | ---- | | ---- | ------- | ---------- | ---- |
| ripgrep | `rg -L -u -tc -n -w '[A-Z]+_SUSPEND'` | 404 | **0.079s** | | ripgrep | `rg -uuu -tc -n -w '[A-Z]+_SUSPEND'` | 388 | **0.096s** |
| [ucg](https://github.com/gvansickle/ucg) | `ucg --type=cc -w '[A-Z]+_SUSPEND'` | 390 | 0.163s | | [ugrep](https://github.com/Genivia/ugrep) | `ugrep -r -n --include='*.c' --include='*.h' -w '[A-Z]+_SUSPEND'` | 388 | 0.493s |
| [GNU grep](https://www.gnu.org/software/grep/) | `egrep -R -n --include='*.c' --include='*.h' -w '[A-Z]+_SUSPEND'` | 404 | 0.611s | | [GNU grep](https://www.gnu.org/software/grep/) | `egrep -r -n --include='*.c' --include='*.h' -w '[A-Z]+_SUSPEND'` | 388 | 0.806s |
(`ucg` [has slightly different behavior in the presence of symbolic links](https://github.com/gvansickle/ucg/issues/106).) And finally, a straight-up comparison between ripgrep, ugrep and GNU grep on a
single large file cached in memory
And finally, a straight-up comparison between ripgrep and GNU grep on a single (~13GB, [`OpenSubtitles.raw.en.gz`](http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2018/mono/OpenSubtitles.raw.en.gz)):
large file (~9.3GB,
[`OpenSubtitles2016.raw.en.gz`](http://opus.lingfil.uu.se/OpenSubtitles2016/mono/OpenSubtitles2016.raw.en.gz)):
| Tool | Command | Line count | Time | | Tool | Command | Line count | Time |
| ---- | ------- | ---------- | ---- | | ---- | ------- | ---------- | ---- |
| ripgrep | `rg -w 'Sherlock [A-Z]\w+'` | 5268 | **2.108s** | | ripgrep | `rg -w 'Sherlock [A-Z]\w+'` | 7882 | **2.769s** |
| [GNU grep](https://www.gnu.org/software/grep/) | `LC_ALL=C egrep -w 'Sherlock [A-Z]\w+'` | 5268 | 7.014s | | [ugrep](https://github.com/Genivia/ugrep) | `ugrep -w 'Sherlock [A-Z]\w+'` | 7882 | 6.802s |
| [GNU grep](https://www.gnu.org/software/grep/) | `LC_ALL=en_US.UTF-8 egrep -w 'Sherlock [A-Z]\w+'` | 7882 | 9.027s |
In the above benchmark, passing the `-n` flag (for showing line numbers) In the above benchmark, passing the `-n` flag (for showing line numbers)
increases the times to `2.640s` for ripgrep and `10.277s` for GNU grep. increases the times to `3.423s` for ripgrep and `13.031s` for GNU grep. ugrep
times are unaffected by the presence or absence of `-n`.
### Why should I use ripgrep? ### Why should I use ripgrep?
@@ -92,11 +91,11 @@ increases the times to `2.640s` for ripgrep and `10.277s` for GNU grep.
[the FAQ](FAQ.md#posix4ever) for more details on whether ripgrep can truly [the FAQ](FAQ.md#posix4ever) for more details on whether ripgrep can truly
replace grep.) replace grep.)
* Like other tools specialized to code search, ripgrep defaults to recursive * Like other tools specialized to code search, ripgrep defaults to recursive
directory search and won't search files ignored by your `.gitignore` files. directory search and won't search files ignored by your
It also ignores hidden and binary files by default. ripgrep also implements `.gitignore`/`.ignore`/`.rgignore` files. It also ignores hidden and binary
full support for `.gitignore`, whereas there are many bugs related to that files by default. ripgrep also implements full support for `.gitignore`,
functionality in other code search tools claiming to provide the same whereas there are many bugs related to that functionality in other code
functionality. search tools claiming to provide the same functionality.
* ripgrep can search specific types of files. For example, `rg -tpy foo` * ripgrep can search specific types of files. For example, `rg -tpy foo`
limits your search to Python files and `rg -Tjs foo` excludes Javascript limits your search to Python files and `rg -Tjs foo` excludes Javascript
files from your search. ripgrep can be taught about new file types with files from your search. ripgrep can be taught about new file types with
@@ -108,13 +107,15 @@ increases the times to `2.640s` for ripgrep and `10.277s` for GNU grep.
* ripgrep has optional support for switching its regex engine to use PCRE2. * ripgrep has optional support for switching its regex engine to use PCRE2.
Among other things, this makes it possible to use look-around and Among other things, this makes it possible to use look-around and
backreferences in your patterns, which are not supported in ripgrep's default backreferences in your patterns, which are not supported in ripgrep's default
regex engine. PCRE2 support is enabled with `-P`. regex engine. PCRE2 support can be enabled with `-P/--pcre2` (use PCRE2
always) or `--auto-hybrid-regex` (use PCRE2 only if needed). An alternative
syntax is provided via the `--engine (default|pcre2|auto-hybrid)` option.
* ripgrep supports searching files in text encodings other than UTF-8, such * ripgrep supports searching files in text encodings other than UTF-8, such
as UTF-16, latin-1, GBK, EUC-JP, Shift_JIS and more. (Some support for as UTF-16, latin-1, GBK, EUC-JP, Shift_JIS and more. (Some support for
automatically detecting UTF-16 is provided. Other text encodings must be automatically detecting UTF-16 is provided. Other text encodings must be
specifically specified with the `-E/--encoding` flag.) specifically specified with the `-E/--encoding` flag.)
* ripgrep supports searching files compressed in a common format (gzip, xz, * ripgrep supports searching files compressed in a common format (brotli,
lzma, bzip2 or lz4) with the `-z/--search-zip` flag. bzip2, gzip, lz4, lzma, xz, or zstandard) with the `-z/--search-zip` flag.
* ripgrep supports arbitrary input preprocessing filters which could be PDF * ripgrep supports arbitrary input preprocessing filters which could be PDF
text extraction, less supported decompression, decrypting, automatic encoding text extraction, less supported decompression, decrypting, automatic encoding
detection and so on. detection and so on.
@@ -148,12 +149,12 @@ or more of the following:
### Is it really faster than everything else? ### Is it really faster than everything else?
Generally, yes. A large number of benchmarks with detailed analysis for each is Generally, yes. A large number of benchmarks with detailed analysis for each is
[available on my blog](http://blog.burntsushi.net/ripgrep/). [available on my blog](https://blog.burntsushi.net/ripgrep/).
Summarizing, ripgrep is fast because: Summarizing, ripgrep is fast because:
* It is built on top of * It is built on top of
[Rust's regex engine](https://github.com/rust-lang-nursery/regex). [Rust's regex engine](https://github.com/rust-lang/regex).
Rust's regex engine uses finite automata, SIMD and aggressive literal Rust's regex engine uses finite automata, SIMD and aggressive literal
optimizations to make searching very fast. (PCRE2 support can be opted into optimizations to make searching very fast. (PCRE2 support can be opted into
with the `-P/--pcre2` flag.) with the `-P/--pcre2` flag.)
@@ -200,22 +201,13 @@ prefer MSVC over GNU, but you'll need to have the [Microsoft VC++ 2015
redistributable](https://www.microsoft.com/en-us/download/details.aspx?id=48145) redistributable](https://www.microsoft.com/en-us/download/details.aspx?id=48145)
installed. installed.
If you're a **macOS Homebrew** or a **Linuxbrew** user, If you're a **macOS Homebrew** or a **Linuxbrew** user, then you can install
then you can install ripgrep either ripgrep from homebrew-core:
from homebrew-core, (compiled with rust stable, no SIMD):
``` ```
$ brew install ripgrep $ brew install ripgrep
``` ```
or you can install a binary compiled with rust nightly (including SIMD and all
optimizations) by utilizing a custom tap:
```
$ brew tap burntsushi/ripgrep https://github.com/BurntSushi/ripgrep.git
$ brew install ripgrep-bin
```
If you're a **MacPorts** user, then you can install ripgrep from the If you're a **MacPorts** user, then you can install ripgrep from the
[official ports](https://www.macports.org/ports.php?by=name&substr=ripgrep): [official ports](https://www.macports.org/ports.php?by=name&substr=ripgrep):
@@ -231,7 +223,7 @@ $ choco install ripgrep
``` ```
If you're a **Windows Scoop** user, then you can install ripgrep from the If you're a **Windows Scoop** user, then you can install ripgrep from the
[official bucket](https://github.com/lukesampson/scoop/blob/master/bucket/ripgrep.json): [official bucket](https://github.com/ScoopInstaller/Main/blob/master/bucket/ripgrep.json):
``` ```
$ scoop install ripgrep $ scoop install ripgrep
@@ -257,23 +249,14 @@ repositories.
$ sudo dnf install ripgrep $ sudo dnf install ripgrep
``` ```
If you're an **openSUSE Leap 15.0** user, you can install ripgrep from the If you're an **openSUSE** user, ripgrep is included in **openSUSE Tumbleweed**
[utilities repo](https://build.opensuse.org/package/show/utilities/ripgrep): and **openSUSE Leap** since 15.1.
```
$ sudo zypper ar https://download.opensuse.org/repositories/utilities/openSUSE_Leap_15.0/utilities.repo
$ sudo zypper install ripgrep
```
If you're an **openSUSE Tumbleweed** user, you can install ripgrep from the
[official repo](http://software.opensuse.org/package/ripgrep):
``` ```
$ sudo zypper install ripgrep $ sudo zypper install ripgrep
``` ```
If you're a **RHEL/CentOS 7** user, you can install ripgrep from If you're a **RHEL/CentOS 7/8** user, you can install ripgrep from
[copr](https://copr.fedorainfracloud.org/coprs/carlwgeorge/ripgrep/): [copr](https://copr.fedorainfracloud.org/coprs/carlwgeorge/ripgrep/):
``` ```
@@ -294,11 +277,11 @@ then ripgrep can be installed using a binary `.deb` file provided in each
[ripgrep release](https://github.com/BurntSushi/ripgrep/releases). [ripgrep release](https://github.com/BurntSushi/ripgrep/releases).
``` ```
$ curl -LO https://github.com/BurntSushi/ripgrep/releases/download/0.10.0/ripgrep_0.10.0_amd64.deb $ curl -LO https://github.com/BurntSushi/ripgrep/releases/download/11.0.2/ripgrep_11.0.2_amd64.deb
$ sudo dpkg -i ripgrep_0.10.0_amd64.deb $ sudo dpkg -i ripgrep_11.0.2_amd64.deb
``` ```
If you run Debian Buster (currently Debian testing) or Debian sid, ripgrep is If you run Debian Buster (currently Debian stable) or Debian sid, ripgrep is
[officially maintained by Debian](https://tracker.debian.org/pkg/rust-ripgrep). [officially maintained by Debian](https://tracker.debian.org/pkg/rust-ripgrep).
``` ```
$ sudo apt-get install ripgrep $ sudo apt-get install ripgrep
@@ -338,6 +321,20 @@ If you're a **NetBSD** user, then you can install ripgrep from
# pkgin install ripgrep # pkgin install ripgrep
``` ```
If you're a **Haiku x86_64** user, then you can install ripgrep from the
[official ports](https://github.com/haikuports/haikuports/tree/master/sys-apps/ripgrep):
```
$ pkgman install ripgrep
```
If you're a **Haiku x86_gcc2** user, then you can install ripgrep from the
same port as Haiku x86_64 using the x86 secondary architecture build:
```
$ pkgman install ripgrep_x86
```
If you're a **Rust programmer**, ripgrep can be installed with `cargo`. If you're a **Rust programmer**, ripgrep can be installed with `cargo`.
* Note that the minimum supported version of Rust for ripgrep is **1.34.0**, * Note that the minimum supported version of Rust for ripgrep is **1.34.0**,
@@ -350,9 +347,6 @@ If you're a **Rust programmer**, ripgrep can be installed with `cargo`.
$ cargo install ripgrep $ cargo install ripgrep
``` ```
ripgrep isn't currently in any other package repositories.
[I'd like to change that](https://github.com/BurntSushi/ripgrep/issues/10).
### Building ### Building
@@ -428,3 +422,11 @@ $ cargo test --all
``` ```
from the repository root. from the repository root.
### Translations
The following is a list of known translations of ripgrep's documentation. These
are unofficially maintained and may not be up to date.
* [Chinese](https://github.com/chinanf-boy/ripgrep-zh#%E6%9B%B4%E6%96%B0-)

81
appveyor.yml
View File

@@ -1,81 +0,0 @@
cache:
- c:\cargo\registry
- c:\cargo\git
init:
- mkdir c:\cargo
- mkdir c:\rustup
- SET PATH=c:\cargo\bin;%PATH%
clone_folder: c:\projects\ripgrep
environment:
CARGO_HOME: "c:\\cargo"
RUSTUP_HOME: "c:\\rustup"
CARGO_TARGET_DIR: "c:\\projects\\ripgrep\\target"
global:
PROJECT_NAME: ripgrep
RUST_BACKTRACE: full
matrix:
- TARGET: x86_64-pc-windows-gnu
CHANNEL: stable
BITS: 64
MSYS2: 1
- TARGET: x86_64-pc-windows-msvc
CHANNEL: stable
BITS: 64
- TARGET: i686-pc-windows-gnu
CHANNEL: stable
BITS: 32
MSYS2: 1
- TARGET: i686-pc-windows-msvc
CHANNEL: stable
BITS: 32
matrix:
fast_finish: true
# Install Rust and Cargo
# (Based on from https://github.com/rust-lang/libc/blob/master/appveyor.yml)
install:
- curl -sSf -o rustup-init.exe https://win.rustup.rs/
- rustup-init.exe -y --default-host %TARGET%
- set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
- if defined MSYS2 set PATH=C:\msys64\mingw%BITS%\bin;%PATH%
- rustc -V
- cargo -V
# Hack to work around a harmless warning in Appveyor builds?
build: false
# Equivalent to Travis' `script` phase
test_script:
- cargo test --verbose --all --features pcre2
before_deploy:
# Generate artifacts for release
- cargo build --release --features pcre2
- mkdir staging
- copy target\release\rg.exe staging
- ps: copy target\release\build\ripgrep-*\out\_rg.ps1 staging
- cd staging
# release zipfile will look like 'ripgrep-1.2.3-x86_64-pc-windows-msvc'
- 7z a ../%PROJECT_NAME%-%APPVEYOR_REPO_TAG_NAME%-%TARGET%.zip *
- appveyor PushArtifact ../%PROJECT_NAME%-%APPVEYOR_REPO_TAG_NAME%-%TARGET%.zip
deploy:
description: 'Automatically deployed release'
# All the zipped artifacts will be deployed
artifact: /.*\.zip/
auth_token:
secure: vv4vBCEosGlyQjaEC1+kraP2P6O4CQSa+Tw50oHWFTGcmuXxaWS0/yEXbxsIRLpw
provider: GitHub
# deploy when a new tag is pushed and only on the stable channel
on:
CHANNEL: stable
appveyor_repo_tag: true
branches:
only:
- /^\d+\.\d+\.\d+$/
- master

22
build.rs
View File

@@ -9,7 +9,7 @@ use clap::Shell;
use app::{RGArg, RGArgKind}; use app::{RGArg, RGArgKind};
#[allow(dead_code)] #[allow(dead_code)]
#[path = "src/app.rs"] #[path = "crates/core/app.rs"]
mod app; mod app;
fn main() { fn main() {
@@ -21,7 +21,8 @@ fn main() {
eprintln!( eprintln!(
"OUT_DIR environment variable not defined. \ "OUT_DIR environment variable not defined. \
Please file a bug: \ Please file a bug: \
https://github.com/BurntSushi/ripgrep/issues/new"); https://github.com/BurntSushi/ripgrep/issues/new"
);
process::exit(1); process::exit(1);
} }
}; };
@@ -85,13 +86,15 @@ fn generate_man_page<P: AsRef<Path>>(outdir: P) -> io::Result<()> {
let githash = git_revision_hash(); let githash = git_revision_hash();
let githash = githash.as_ref().map(|x| &**x); let githash = githash.as_ref().map(|x| &**x);
tpl = tpl.replace("{VERSION}", &app::long_version(githash)); tpl = tpl.replace("{VERSION}", &app::long_version(githash, false));
File::create(&txt_path)?.write_all(tpl.as_bytes())?; File::create(&txt_path)?.write_all(tpl.as_bytes())?;
let result = process::Command::new("a2x") let result = process::Command::new("a2x")
.arg("--no-xmllint") .arg("--no-xmllint")
.arg("--doctype").arg("manpage") .arg("--doctype")
.arg("--format").arg("manpage") .arg("manpage")
.arg("--format")
.arg("manpage")
.arg(&txt_path) .arg(&txt_path)
.spawn()? .spawn()?
.wait()?; .wait()?;
@@ -114,7 +117,7 @@ fn formatted_options() -> io::Result<String> {
// ripgrep only has two positional arguments, and probably will only // ripgrep only has two positional arguments, and probably will only
// ever have two positional arguments, so we just hardcode them into // ever have two positional arguments, so we just hardcode them into
// the template. // the template.
if let app::RGArgKind::Positional{..} = arg.kind { if let app::RGArgKind::Positional { .. } = arg.kind {
continue; continue;
} }
formatted.push(formatted_arg(&arg)?); formatted.push(formatted_arg(&arg)?);
@@ -124,7 +127,9 @@ fn formatted_options() -> io::Result<String> {
fn formatted_arg(arg: &RGArg) -> io::Result<String> { fn formatted_arg(arg: &RGArg) -> io::Result<String> {
match arg.kind { match arg.kind {
RGArgKind::Positional{..} => panic!("unexpected positional argument"), RGArgKind::Positional { .. } => {
panic!("unexpected positional argument")
}
RGArgKind::Switch { long, short, multiple } => { RGArgKind::Switch { long, short, multiple } => {
let mut out = vec![]; let mut out = vec![];
@@ -163,7 +168,8 @@ fn formatted_arg(arg: &RGArg) -> io::Result<String> {
} }
fn formatted_doc_txt(arg: &RGArg) -> io::Result<String> { fn formatted_doc_txt(arg: &RGArg) -> io::Result<String> {
let paragraphs: Vec<String> = arg.doc_long let paragraphs: Vec<String> = arg
.doc_long
.replace("{", "&#123;") .replace("{", "&#123;")
.replace("}", r"&#125;") .replace("}", r"&#125;")
.split("\n\n") .split("\n\n")

61
ci/before_deploy.sh
View File

@@ -1,61 +0,0 @@
#!/bin/bash
# package the build artifacts
set -ex
. "$(dirname $0)/utils.sh"
# Generate artifacts for release
mk_artifacts() {
if is_arm; then
cargo build --target "$TARGET" --release
else
# Technically, MUSL builds will force PCRE2 to get statically compiled,
# but we also want PCRE2 statically build for macOS binaries.
PCRE2_SYS_STATIC=1 cargo build --target "$TARGET" --release --features 'pcre2'
fi
}
mk_tarball() {
# When cross-compiling, use the right `strip` tool on the binary.
local gcc_prefix="$(gcc_prefix)"
# Create a temporary dir that contains our staging area.
# $tmpdir/$name is what eventually ends up as the deployed archive.
local tmpdir="$(mktemp -d)"
local name="${PROJECT_NAME}-${TRAVIS_TAG}-${TARGET}"
local staging="$tmpdir/$name"
mkdir -p "$staging"/{complete,doc}
# The deployment directory is where the final archive will reside.
# This path is known by the .travis.yml configuration.
local out_dir="$(pwd)/deployment"
mkdir -p "$out_dir"
# Find the correct (most recent) Cargo "out" directory. The out directory
# contains shell completion files and the man page.
local cargo_out_dir="$(cargo_out_dir "target/$TARGET")"
# Copy the ripgrep binary and strip it.
cp "target/$TARGET/release/rg" "$staging/rg"
"${gcc_prefix}strip" "$staging/rg"
# Copy the licenses and README.
cp {README.md,UNLICENSE,COPYING,LICENSE-MIT} "$staging/"
# Copy documentation and man page.
cp {CHANGELOG.md,FAQ.md,GUIDE.md} "$staging/doc/"
if command -V a2x 2>&1 > /dev/null; then
# The man page should only exist if we have asciidoc installed.
cp "$cargo_out_dir/rg.1" "$staging/doc/"
fi
# Copy shell completion files.
cp "$cargo_out_dir"/{rg.bash,rg.fish,_rg.ps1} "$staging/complete/"
cp complete/_rg "$staging/complete/"
(cd "$tmpdir" && tar czf "$out_dir/$name.tar.gz" "$name")
rm -rf "$tmpdir"
}
main() {
mk_artifacts
mk_tarball
}
main

15
ci/build_deb.sh → ci/build-deb
View File

@@ -1,6 +1,7 @@
#!/bin/bash #!/bin/bash
set -e set -e
D="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
# This script builds a binary dpkg for Debian based distros. It does not # This script builds a binary dpkg for Debian based distros. It does not
# currently run in CI, and is instead run manually and the resulting dpkg is # currently run in CI, and is instead run manually and the resulting dpkg is
@@ -23,20 +24,12 @@ fi
# the deb, which knows where to look. # the deb, which knows where to look.
DEPLOY_DIR=deployment/deb DEPLOY_DIR=deployment/deb
OUT_DIR="$("$D"/cargo-out-dir target/debug/)"
mkdir -p "$DEPLOY_DIR" mkdir -p "$DEPLOY_DIR"
cargo build cargo build
# Find and copy man page. # Copy man page and shell completions.
manpage="$(find ./target/debug -name rg.1 -print0 | xargs -0 ls -t | head -n1)" cp "$OUT_DIR"/{rg.1,rg.bash,rg.fish,_rg} "$DEPLOY_DIR/"
cp "$manpage" "$DEPLOY_DIR/"
# Do the same for shell completions.
compbash="$(find ./target/debug -name rg.bash -print0 | xargs -0 ls -t | head -n1)"
cp "$compbash" "$DEPLOY_DIR/"
compfish="$(find ./target/debug -name rg.fish -print0 | xargs -0 ls -t | head -n1)"
cp "$compfish" "$DEPLOY_DIR/"
compzsh="complete/_rg"
cp "$compzsh" "$DEPLOY_DIR/"
# Since we're distributing the dpkg, we don't know whether the user will have # Since we're distributing the dpkg, we don't know whether the user will have
# PCRE2 installed, so just do a static build. # PCRE2 installed, so just do a static build.

19
ci/cargo-out-dir Executable file
View File

@@ -0,0 +1,19 @@
#!/bin/bash
# Finds Cargo's `OUT_DIR` directory from the most recent build.
#
# This requires one parameter corresponding to the target directory
# to search for the build output.
if [ $# != 1 ]; then
echo "Usage: $(basename "$0") <target-dir>" >&2
exit 2
fi
# This works by finding the most recent stamp file, which is produced by
# every ripgrep build.
target_dir="$1"
find "$target_dir" -name ripgrep-stamp -print0 \
| xargs -0 ls -t \
| head -n1 \
| xargs dirname

24
ci/docker/README.md Normal file
View File

@@ -0,0 +1,24 @@
These are Docker images used for cross compilation in CI builds (or locally)
via the [Cross](https://github.com/rust-embedded/cross) tool.
The Cross tool actually provides its own Docker images, and all Docker images
in this directory are derived from one of them. We provide our own in order
to customize the environment. For example, we need to install some things like
`asciidoc` in order to generate man pages. We also install compression tools
like `xz` so that tests for the `-z/--search-zip` flag are run.
If you make a change to a Docker image, then you can re-build it. `cd` into the
directory containing the `Dockerfile` and run:
$ cd x86_64-unknown-linux-musl
$ ./build
At this point, subsequent uses of `cross` will now use your built image since
Docker prefers local images over remote images. In order to make these changes
stick, they need to be pushed to Docker Hub:
$ docker push burntsushi/cross:x86_64-unknown-linux-musl
Of course, only I (BurntSushi) can push to that location. To make `cross` use
a different location, then edit `Cross.toml` in the root of this repo to use
a different image name for the desired target.

4
ci/docker/arm-unknown-linux-gnueabihf/Dockerfile Normal file
View File

@@ -0,0 +1,4 @@
FROM rustembedded/cross:arm-unknown-linux-gnueabihf
COPY stage/ubuntu-install-packages /
RUN /ubuntu-install-packages

5
ci/docker/arm-unknown-linux-gnueabihf/build Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
mkdir -p stage
cp ../../ubuntu-install-packages ./stage/
docker build -t burntsushi/cross:arm-unknown-linux-gnueabihf .

4
ci/docker/i686-unknown-linux-gnu/Dockerfile Normal file
View File

@@ -0,0 +1,4 @@
FROM rustembedded/cross:i686-unknown-linux-gnu
COPY stage/ubuntu-install-packages /
RUN /ubuntu-install-packages

5
ci/docker/i686-unknown-linux-gnu/build Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
mkdir -p stage
cp ../../ubuntu-install-packages ./stage/
docker build -t burntsushi/cross:i686-unknown-linux-gnu .

4
ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile Normal file
View File

@@ -0,0 +1,4 @@
FROM rustembedded/cross:mips64-unknown-linux-gnuabi64
COPY stage/ubuntu-install-packages /
RUN /ubuntu-install-packages

5
ci/docker/mips64-unknown-linux-gnuabi64/build Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
mkdir -p stage
cp ../../ubuntu-install-packages ./stage/
docker build -t burntsushi/cross:mips64-unknown-linux-gnuabi64 .

4
ci/docker/x86_64-unknown-linux-musl/Dockerfile Normal file
View File

@@ -0,0 +1,4 @@
FROM rustembedded/cross:x86_64-unknown-linux-musl
COPY stage/ubuntu-install-packages /
RUN /ubuntu-install-packages

5
ci/docker/x86_64-unknown-linux-musl/build Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
mkdir -p stage
cp ../../ubuntu-install-packages ./stage/
docker build -t burntsushi/cross:x86_64-unknown-linux-musl .

61
ci/install.sh
View File

@@ -1,61 +0,0 @@
#!/bin/bash
# install stuff needed for the `script` phase
# Where rustup gets installed.
export PATH="$PATH:$HOME/.cargo/bin"
set -ex
. "$(dirname $0)/utils.sh"
install_rustup() {
curl https://sh.rustup.rs -sSf \
| sh -s -- -y --default-toolchain="$TRAVIS_RUST_VERSION"
rustc -V
cargo -V
}
install_targets() {
if [ $(host) != "$TARGET" ]; then
rustup target add $TARGET
fi
}
install_osx_dependencies() {
if ! is_osx; then
return
fi
brew install asciidoc docbook-xsl
}
configure_cargo() {
local prefix=$(gcc_prefix)
if [ -n "${prefix}" ]; then
local gcc_suffix=
if [ -n "$GCC_VERSION" ]; then
gcc_suffix="-$GCC_VERSION"
fi
local gcc="${prefix}gcc${gcc_suffix}"
# information about the cross compiler
"${gcc}" -v
# tell cargo which linker to use for cross compilation
mkdir -p .cargo
cat >>.cargo/config <<EOF
[target.$TARGET]
linker = "${gcc}"
EOF
fi
}
main() {
install_osx_dependencies
install_rustup
install_targets
configure_cargo
}
main

3
ci/macos-install-packages Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/sh
brew install asciidoc docbook-xsl

50
ci/script.sh
View File

@@ -1,50 +0,0 @@
#!/bin/bash
# build, test and generate docs in this phase
set -ex
. "$(dirname $0)/utils.sh"
main() {
# Test a normal debug build.
if is_arm; then
cargo build --target "$TARGET" --verbose
else
cargo build --target "$TARGET" --verbose --all --features 'pcre2'
fi
# Show the output of the most recent build.rs stderr.
set +x
stderr="$(find "target/$TARGET/debug" -name stderr -print0 | xargs -0 ls -t | head -n1)"
if [ -s "$stderr" ]; then
echo "===== $stderr ====="
cat "$stderr"
echo "====="
fi
set -x
# sanity check the file type
file target/"$TARGET"/debug/rg
# Check that we've generated man page and other shell completions.
outdir="$(cargo_out_dir "target/$TARGET/debug")"
file "$outdir/rg.bash"
file "$outdir/rg.fish"
file "$outdir/_rg.ps1"
file "$outdir/rg.1"
# Apparently tests don't work on arm, so just bail now. I guess we provide
# ARM releases on a best effort basis?
if is_arm; then
return 0
fi
# Test that zsh completions are in sync with ripgrep's actual args.
"$(dirname "${0}")/test_complete.sh"
# Run tests for ripgrep and all sub-crates.
cargo test --target "$TARGET" --verbose --all --features 'pcre2'
}
main

0
ci/sha256.sh → ci/sha256-releases
View File

4
ci/test_complete.sh → ci/test-complete
View File

@@ -18,7 +18,7 @@ get_comp_args() {
main() { main() {
local diff local diff
local rg="${0:a:h}/../target/${TARGET:-}/release/rg" local rg="${0:a:h}/../${TARGET_DIR:-target}/release/rg"
local _rg="${0:a:h}/../complete/_rg" local _rg="${0:a:h}/../complete/_rg"
local -a help_args comp_args local -a help_args comp_args
@@ -44,7 +44,7 @@ main() {
# Occasionally we may have to handle some manually, however # Occasionally we may have to handle some manually, however
help_args=( ${(f)"$( help_args=( ${(f)"$(
$rg --help | $rg --help |
$rg -i -- '^\s+--?[a-z0-9]|--[imnp]' | $rg -i -- '^\s+--?[a-z0-9]|--[a-z]' |
$rg -ior '$1' -- $'[\t /\"\'`.,](-[a-z0-9]|--[a-z0-9-]+)\\b' | $rg -ior '$1' -- $'[\t /\"\'`.,](-[a-z0-9]|--[a-z0-9-]+)\\b' |
$rg -v -- --print0 | # False positives $rg -v -- --print0 | # False positives
sort -u sort -u

6
ci/ubuntu-install-packages Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/sh
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libxslt1-dev asciidoc docbook-xsl xsltproc libxml2-utils \
zsh xz-utils liblz4-tool musl-tools

25
ci/utils.sh
View File

@@ -55,6 +55,13 @@ gcc_prefix() {
esac esac
} }
is_musl() {
case "$TARGET" in
*-musl) return 0 ;;
*) return 1 ;;
esac
}
is_x86() { is_x86() {
case "$(architecture)" in case "$(architecture)" in
amd64|i386) return 0 ;; amd64|i386) return 0 ;;
@@ -62,6 +69,13 @@ is_x86() {
esac esac
} }
is_x86_64() {
case "$(architecture)" in
amd64) return 0 ;;
*) return 1 ;;
esac
}
is_arm() { is_arm() {
case "$(architecture)" in case "$(architecture)" in
armhf) return 0 ;; armhf) return 0 ;;
@@ -82,3 +96,14 @@ is_osx() {
*) return 1 ;; *) return 1 ;;
esac esac
} }
builder() {
if is_musl && is_x86_64; then
# cargo install cross
# To work around https://github.com/rust-embedded/cross/issues/357
cargo install --git https://github.com/rust-embedded/cross --force
echo "cross"
else
echo "cargo"
fi
}

36
complete/_rg
View File

@@ -3,7 +3,7 @@
## ##
# zsh completion function for ripgrep # zsh completion function for ripgrep
# #
# Run ci/test_complete.sh after building to ensure that the options supported by # Run ci/test-complete after building to ensure that the options supported by
# this function stay in synch with the `rg` binary. # this function stay in synch with the `rg` binary.
# #
# For convenience, a completion reference guide is included at the bottom of # For convenience, a completion reference guide is included at the bottom of
@@ -72,11 +72,19 @@ _rg() {
+ '(count)' # Counting options + '(count)' # Counting options
{-c,--count}'[only show count of matching lines for each file]' {-c,--count}'[only show count of matching lines for each file]'
'--count-matches[only show count of individual matches for each file]' '--count-matches[only show count of individual matches for each file]'
'--include-zero[include files with zero matches in summary]'
+ '(encoding)' # Encoding options + '(encoding)' # Encoding options
{-E+,--encoding=}'[specify text encoding of files to search]: :_rg_encodings' {-E+,--encoding=}'[specify text encoding of files to search]: :_rg_encodings'
$no'--no-encoding[use default text encoding]' $no'--no-encoding[use default text encoding]'
+ '(engine)' # Engine choice options
'--engine=[select which regex engine to use]:when:((
default\:"use default engine"
pcre2\:"identical to --pcre2"
auto\:"identical to --auto-hybrid-regex"
))'
+ file # File-input options + file # File-input options
'(1)*'{-f+,--file=}'[specify file containing patterns to search for]: :_files' '(1)*'{-f+,--file=}'[specify file containing patterns to search for]: :_files'
@@ -104,6 +112,10 @@ _rg() {
'*'{-g+,--glob=}'[include/exclude files matching specified glob]:glob' '*'{-g+,--glob=}'[include/exclude files matching specified glob]:glob'
'*--iglob=[include/exclude files matching specified case-insensitive glob]:glob' '*--iglob=[include/exclude files matching specified case-insensitive glob]:glob'
+ '(glob-case-insensitive)' # File-glob case sensitivity options
'--glob-case-insensitive[treat -g/--glob patterns case insensitively]'
$no'--no-glob-case-insensitive[treat -g/--glob patterns case sensitively]'
+ '(heading)' # Heading options + '(heading)' # Heading options
'(pretty-vimgrep)--heading[show matches grouped by file name]' '(pretty-vimgrep)--heading[show matches grouped by file name]'
"(pretty-vimgrep)--no-heading[don't show matches grouped by file name]" "(pretty-vimgrep)--no-heading[don't show matches grouped by file name]"
@@ -124,6 +136,10 @@ _rg() {
'--ignore-file-case-insensitive[process ignore files case insensitively]' '--ignore-file-case-insensitive[process ignore files case insensitively]'
$no'--no-ignore-file-case-insensitive[process ignore files case sensitively]' $no'--no-ignore-file-case-insensitive[process ignore files case sensitively]'
+ '(ignore-exclude)' # Local exclude (ignore)-file options
"--no-ignore-exclude[don't respect local exclude (ignore) files]"
$no'--ignore-exclude[respect local exclude (ignore) files]'
+ '(ignore-global)' # Global ignore-file options + '(ignore-global)' # Global ignore-file options
"--no-ignore-global[don't respect global ignore files]" "--no-ignore-global[don't respect global ignore files]"
$no'--ignore-global[respect global ignore files]' $no'--ignore-global[respect global ignore files]'
@@ -136,10 +152,18 @@ _rg() {
"--no-ignore-vcs[don't respect version control ignore files]" "--no-ignore-vcs[don't respect version control ignore files]"
$no'--ignore-vcs[respect version control ignore files]' $no'--ignore-vcs[respect version control ignore files]'
+ '(ignore-dot)' # .ignore-file options + '(require-git)' # git specific settings
"--no-require-git[don't require git repository to respect gitignore rules]"
$no'--require-git[require git repository to respect gitignore rules]'
+ '(ignore-dot)' # .ignore options
"--no-ignore-dot[don't respect .ignore files]" "--no-ignore-dot[don't respect .ignore files]"
$no'--ignore-dot[respect .ignore files]' $no'--ignore-dot[respect .ignore files]'
+ '(ignore-files)' # custom global ignore file options
"--no-ignore-files[don't respect --ignore-file flags]"
$no'--ignore-files[respect --ignore-file files]'
+ '(json)' # JSON options + '(json)' # JSON options
'--json[output results in JSON Lines format]' '--json[output results in JSON Lines format]'
$no"--no-json[don't output results in JSON Lines format]" $no"--no-json[don't output results in JSON Lines format]"
@@ -259,6 +283,10 @@ _rg() {
{-w,--word-regexp}'[only show matches surrounded by word boundaries]' {-w,--word-regexp}'[only show matches surrounded by word boundaries]'
{-x,--line-regexp}'[only show matches surrounded by line boundaries]' {-x,--line-regexp}'[only show matches surrounded by line boundaries]'
+ '(unicode)' # Unicode options
$no'--unicode[enable Unicode mode]'
'--no-unicode[disable Unicode mode]'
+ '(zip)' # Compression options + '(zip)' # Compression options
'(--pre)'{-z,--search-zip}'[search in compressed files]' '(--pre)'{-z,--search-zip}'[search in compressed files]'
$no"--no-search-zip[don't search in compressed files]" $no"--no-search-zip[don't search in compressed files]"
@@ -273,7 +301,9 @@ _rg() {
))' ))'
'*--colors=[specify color and style settings]: :->colorspec' '*--colors=[specify color and style settings]: :->colorspec'
'--context-separator=[specify string used to separate non-continuous context lines in output]:separator' '--context-separator=[specify string used to separate non-continuous context lines in output]:separator'
$no"--no-context-separator[don't print context separators]"
'--debug[show debug messages]' '--debug[show debug messages]'
'--trace[show more verbose debug messages]'
'--dfa-size-limit=[specify upper size limit of generated DFA]:DFA size (bytes)' '--dfa-size-limit=[specify upper size limit of generated DFA]:DFA size (bytes)'
"(1 stats)--files[show each file that would be searched (but don't search)]" "(1 stats)--files[show each file that would be searched (but don't search)]"
'*--ignore-file=[specify additional ignore file]:ignore file:_files' '*--ignore-file=[specify additional ignore file]:ignore file:_files'
@@ -293,7 +323,7 @@ _rg() {
'(--type-list)*: :_files' '(--type-list)*: :_files'
) )
# This is used with test_complete.sh to verify that there are no options # This is used with test-complete to verify that there are no options
# listed in the help output that aren't also defined here # listed in the help output that aren't also defined here
[[ $_RG_COMPLETE_LIST_ARGS == (1|t*|y*) ]] && { [[ $_RG_COMPLETE_LIST_ARGS == (1|t*|y*) ]] && {
print -rl - $args print -rl - $args

8
grep-cli/Cargo.toml → crates/cli/Cargo.toml
View File

@@ -1,20 +1,20 @@
[package] [package]
name = "grep-cli" name = "grep-cli"
version = "0.1.1" #:version version = "0.1.4" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"] authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """ description = """
Utilities for search oriented command line applications. Utilities for search oriented command line applications.
""" """
documentation = "https://docs.rs/grep-cli" documentation = "https://docs.rs/grep-cli"
homepage = "https://github.com/BurntSushi/ripgrep" homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/cli"
repository = "https://github.com/BurntSushi/ripgrep" repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/cli"
readme = "README.md" readme = "README.md"
keywords = ["regex", "grep", "cli", "utility", "util"] keywords = ["regex", "grep", "cli", "utility", "util"]
license = "Unlicense/MIT" license = "Unlicense/MIT"
[dependencies] [dependencies]
atty = "0.2.11" atty = "0.2.11"
bstr = "0.1.2" bstr = "0.2.0"
globset = { version = "0.4.3", path = "../globset" } globset = { version = "0.4.3", path = "../globset" }
lazy_static = "1.1.0" lazy_static = "1.1.0"
log = "0.4.5" log = "0.4.5"

0
globset/LICENSE-MIT → crates/cli/LICENSE-MIT
View File

0
grep-cli/README.md → crates/cli/README.md
View File

0
globset/UNLICENSE → crates/cli/UNLICENSE
View File

30
grep-cli/src/decompress.rs → crates/cli/src/decompress.rs
View File

@@ -38,10 +38,7 @@ impl Default for DecompressionMatcherBuilder {
impl DecompressionMatcherBuilder { impl DecompressionMatcherBuilder {
/// Create a new builder for configuring a decompression matcher. /// Create a new builder for configuring a decompression matcher.
pub fn new() -> DecompressionMatcherBuilder { pub fn new() -> DecompressionMatcherBuilder {
DecompressionMatcherBuilder { DecompressionMatcherBuilder { commands: vec![], defaults: true }
commands: vec![],
defaults: true,
}
} }
/// Build a matcher for determining how to decompress files. /// Build a matcher for determining how to decompress files.
@@ -49,12 +46,11 @@ impl DecompressionMatcherBuilder {
/// If there was a problem compiling the matcher, then an error is /// If there was a problem compiling the matcher, then an error is
/// returned. /// returned.
pub fn build(&self) -> Result<DecompressionMatcher, CommandError> { pub fn build(&self) -> Result<DecompressionMatcher, CommandError> {
let defaults = let defaults = if !self.defaults {
if !self.defaults { vec![]
vec![] } else {
} else { default_decompression_commands()
default_decompression_commands() };
};
let mut glob_builder = GlobSetBuilder::new(); let mut glob_builder = GlobSetBuilder::new();
let mut commands = vec![]; let mut commands = vec![];
for decomp_cmd in defaults.iter().chain(&self.commands) { for decomp_cmd in defaults.iter().chain(&self.commands) {
@@ -93,17 +89,15 @@ impl DecompressionMatcherBuilder {
program: P, program: P,
args: I, args: I,
) -> &mut DecompressionMatcherBuilder ) -> &mut DecompressionMatcherBuilder
where P: AsRef<OsStr>, where
I: IntoIterator<Item=A>, P: AsRef<OsStr>,
A: AsRef<OsStr>, I: IntoIterator<Item = A>,
A: AsRef<OsStr>,
{ {
let glob = glob.to_string(); let glob = glob.to_string();
let bin = program.as_ref().to_os_string(); let bin = program.as_ref().to_os_string();
let args = args let args =
.into_iter() args.into_iter().map(|a| a.as_ref().to_os_string()).collect();
.map(|a| a.as_ref().to_os_string())
.collect();
self.commands.push(DecompressionCommand { glob, bin, args }); self.commands.push(DecompressionCommand { glob, bin, args });
self self
} }

99
grep-cli/src/escape.rs → crates/cli/src/escape.rs
View File

@@ -1,7 +1,7 @@
use std::ffi::OsStr; use std::ffi::OsStr;
use std::str; use std::str;
use bstr::{BStr, BString}; use bstr::{ByteSlice, ByteVec};
/// A single state in the state machine used by `unescape`. /// A single state in the state machine used by `unescape`.
#[derive(Clone, Copy, Eq, PartialEq)] #[derive(Clone, Copy, Eq, PartialEq)]
@@ -38,7 +38,6 @@ enum State {
/// assert_eq!(r"foo\nbar\xFFbaz", escape(b"foo\nbar\xFFbaz")); /// assert_eq!(r"foo\nbar\xFFbaz", escape(b"foo\nbar\xFFbaz"));
/// ``` /// ```
pub fn escape(bytes: &[u8]) -> String { pub fn escape(bytes: &[u8]) -> String {
let bytes = BStr::new(bytes);
let mut escaped = String::new(); let mut escaped = String::new();
for (s, e, ch) in bytes.char_indices() { for (s, e, ch) in bytes.char_indices() {
if ch == '\u{FFFD}' { if ch == '\u{FFFD}' {
@@ -56,7 +55,7 @@ pub fn escape(bytes: &[u8]) -> String {
/// ///
/// This is like [`escape`](fn.escape.html), but accepts an OS string. /// This is like [`escape`](fn.escape.html), but accepts an OS string.
pub fn escape_os(string: &OsStr) -> String { pub fn escape_os(string: &OsStr) -> String {
escape(BString::from_os_str_lossy(string).as_bytes()) escape(Vec::from_os_str_lossy(string).as_bytes())
} }
/// Unescapes a string. /// Unescapes a string.
@@ -96,51 +95,61 @@ pub fn unescape(s: &str) -> Vec<u8> {
let mut state = Literal; let mut state = Literal;
for c in s.chars() { for c in s.chars() {
match state { match state {
Escape => { Escape => match c {
match c { '\\' => {
'\\' => { bytes.push(b'\\'); state = Literal; } bytes.push(b'\\');
'n' => { bytes.push(b'\n'); state = Literal; } state = Literal;
'r' => { bytes.push(b'\r'); state = Literal; }
't' => { bytes.push(b'\t'); state = Literal; }
'x' => { state = HexFirst; }
c => {
bytes.extend(format!(r"\{}", c).into_bytes());
state = Literal;
}
} }
} 'n' => {
HexFirst => { bytes.push(b'\n');
match c { state = Literal;
'0'...'9' | 'A'...'F' | 'a'...'f' => {
state = HexSecond(c);
}
c => {
bytes.extend(format!(r"\x{}", c).into_bytes());
state = Literal;
}
} }
} 'r' => {
HexSecond(first) => { bytes.push(b'\r');
match c { state = Literal;
'0'...'9' | 'A'...'F' | 'a'...'f' => {
let ordinal = format!("{}{}", first, c);
let byte = u8::from_str_radix(&ordinal, 16).unwrap();
bytes.push(byte);
state = Literal;
}
c => {
let original = format!(r"\x{}{}", first, c);
bytes.extend(original.into_bytes());
state = Literal;
}
} }
} 't' => {
Literal => { bytes.push(b'\t');
match c { state = Literal;
'\\' => { state = Escape; }
c => { bytes.extend(c.to_string().as_bytes()); }
} }
} 'x' => {
state = HexFirst;
}
c => {
bytes.extend(format!(r"\{}", c).into_bytes());
state = Literal;
}
},
HexFirst => match c {
'0'..='9' | 'A'..='F' | 'a'..='f' => {
state = HexSecond(c);
}
c => {
bytes.extend(format!(r"\x{}", c).into_bytes());
state = Literal;
}
},
HexSecond(first) => match c {
'0'..='9' | 'A'..='F' | 'a'..='f' => {
let ordinal = format!("{}{}", first, c);
let byte = u8::from_str_radix(&ordinal, 16).unwrap();
bytes.push(byte);
state = Literal;
}
c => {
let original = format!(r"\x{}{}", first, c);
bytes.extend(original.into_bytes());
state = Literal;
}
},
Literal => match c {
'\\' => {
state = Escape;
}
c => {
bytes.extend(c.to_string().as_bytes());
}
},
} }
} }
match state { match state {
@@ -174,7 +183,7 @@ fn escape_char(cp: char, into: &mut String) {
/// Adds the given byte to the given string, escaping it if necessary. /// Adds the given byte to the given string, escaping it if necessary.
fn escape_byte(byte: u8, into: &mut String) { fn escape_byte(byte: u8, into: &mut String) {
match byte { match byte {
0x21...0x5B | 0x5D...0x7D => into.push(byte as char), 0x21..=0x5B | 0x5D..=0x7D => into.push(byte as char),
b'\n' => into.push_str(r"\n"), b'\n' => into.push_str(r"\n"),
b'\r' => into.push_str(r"\r"), b'\r' => into.push_str(r"\r"),
b'\t' => into.push_str(r"\t"), b'\t' => into.push_str(r"\t"),

50
grep-cli/src/human.rs → crates/cli/src/human.rs
View File

@@ -46,7 +46,9 @@ impl ParseSizeError {
} }
impl error::Error for ParseSizeError { impl error::Error for ParseSizeError {
fn description(&self) -> &str { "invalid size" } fn description(&self) -> &str {
"invalid size"
}
} }
impl fmt::Display for ParseSizeError { impl fmt::Display for ParseSizeError {
@@ -54,26 +56,19 @@ impl fmt::Display for ParseSizeError {
use self::ParseSizeErrorKind::*; use self::ParseSizeErrorKind::*;
match self.kind { match self.kind {
InvalidFormat => { InvalidFormat => write!(
write!( f,
f, "invalid format for size '{}', which should be a sequence \
"invalid format for size '{}', which should be a sequence \
of digits followed by an optional 'K', 'M' or 'G' \ of digits followed by an optional 'K', 'M' or 'G' \
suffix", suffix",
self.original self.original
) ),
} InvalidInt(ref err) => write!(
InvalidInt(ref err) => { f,
write!( "invalid integer found in size '{}': {}",
f, self.original, err
"invalid integer found in size '{}': {}", ),
self.original, Overflow => write!(f, "size too big in '{}'", self.original),
err
)
}
Overflow => {
write!(f, "size too big in '{}'", self.original)
}
} }
} }
} }
@@ -104,17 +99,16 @@ pub fn parse_human_readable_size(size: &str) -> Result<u64, ParseSizeError> {
Some(caps) => caps, Some(caps) => caps,
None => return Err(ParseSizeError::format(size)), None => return Err(ParseSizeError::format(size)),
}; };
let value: u64 = caps[1].parse().map_err(|err| { let value: u64 =
ParseSizeError::int(size, err) caps[1].parse().map_err(|err| ParseSizeError::int(size, err))?;
})?;
let suffix = match caps.get(2) { let suffix = match caps.get(2) {
None => return Ok(value), None => return Ok(value),
Some(cap) => cap.as_str(), Some(cap) => cap.as_str(),
}; };
let bytes = match suffix { let bytes = match suffix {
"K" => value.checked_mul(1<<10), "K" => value.checked_mul(1 << 10),
"M" => value.checked_mul(1<<20), "M" => value.checked_mul(1 << 20),
"G" => value.checked_mul(1<<30), "G" => value.checked_mul(1 << 30),
// Because if the regex matches this group, it must be [KMG]. // Because if the regex matches this group, it must be [KMG].
_ => unreachable!(), _ => unreachable!(),
}; };
@@ -134,19 +128,19 @@ mod tests {
#[test] #[test]
fn suffix_k() { fn suffix_k() {
let x = parse_human_readable_size("123K").unwrap(); let x = parse_human_readable_size("123K").unwrap();
assert_eq!(123 * (1<<10), x); assert_eq!(123 * (1 << 10), x);
} }
#[test] #[test]
fn suffix_m() { fn suffix_m() {
let x = parse_human_readable_size("123M").unwrap(); let x = parse_human_readable_size("123M").unwrap();
assert_eq!(123 * (1<<20), x); assert_eq!(123 * (1 << 20), x);
} }
#[test] #[test]
fn suffix_g() { fn suffix_g() {
let x = parse_human_readable_size("123G").unwrap(); let x = parse_human_readable_size("123G").unwrap();
assert_eq!(123 * (1<<30), x); assert_eq!(123 * (1 << 30), x);
} }
#[test] #[test]

16
grep-cli/src/lib.rs → crates/cli/src/lib.rs
View File

@@ -179,20 +179,18 @@ mod process;
mod wtr; mod wtr;
pub use decompress::{ pub use decompress::{
DecompressionMatcher, DecompressionMatcherBuilder, DecompressionMatcher, DecompressionMatcherBuilder, DecompressionReader,
DecompressionReader, DecompressionReaderBuilder, DecompressionReaderBuilder,
}; };
pub use escape::{escape, escape_os, unescape, unescape_os}; pub use escape::{escape, escape_os, unescape, unescape_os};
pub use human::{ParseSizeError, parse_human_readable_size}; pub use human::{parse_human_readable_size, ParseSizeError};
pub use pattern::{ pub use pattern::{
InvalidPatternError, pattern_from_bytes, pattern_from_os, patterns_from_path,
pattern_from_os, pattern_from_bytes, patterns_from_reader, patterns_from_stdin, InvalidPatternError,
patterns_from_path, patterns_from_reader, patterns_from_stdin,
}; };
pub use process::{CommandError, CommandReader, CommandReaderBuilder}; pub use process::{CommandError, CommandReader, CommandReaderBuilder};
pub use wtr::{ pub use wtr::{
StandardStream, stdout, stdout_buffered_block, stdout_buffered_line, StandardStream,
stdout, stdout_buffered_line, stdout_buffered_block,
}; };
/// Returns true if and only if stdin is believed to be readable. /// Returns true if and only if stdin is believed to be readable.
@@ -205,8 +203,8 @@ pub use wtr::{
pub fn is_readable_stdin() -> bool { pub fn is_readable_stdin() -> bool {
#[cfg(unix)] #[cfg(unix)]
fn imp() -> bool { fn imp() -> bool {
use std::os::unix::fs::FileTypeExt;
use same_file::Handle; use same_file::Handle;
use std::os::unix::fs::FileTypeExt;
let ft = match Handle::stdin().and_then(|h| h.as_file().metadata()) { let ft = match Handle::stdin().and_then(|h| h.as_file().metadata()) {
Err(_) => return false, Err(_) => return false,

54
grep-cli/src/pattern.rs → crates/cli/src/pattern.rs
View File

@@ -2,10 +2,12 @@ use std::error;
use std::ffi::OsStr; use std::ffi::OsStr;
use std::fmt; use std::fmt;
use std::fs::File; use std::fs::File;
use std::io::{self, BufRead}; use std::io;
use std::path::Path; use std::path::Path;
use std::str; use std::str;
use bstr::io::BufReadExt;
use escape::{escape, escape_os}; use escape::{escape, escape_os};
/// An error that occurs when a pattern could not be converted to valid UTF-8. /// An error that occurs when a pattern could not be converted to valid UTF-8.
@@ -27,7 +29,9 @@ impl InvalidPatternError {
} }
impl error::Error for InvalidPatternError { impl error::Error for InvalidPatternError {
fn description(&self) -> &str { "invalid pattern" } fn description(&self) -> &str {
"invalid pattern"
}
} }
impl fmt::Display for InvalidPatternError { impl fmt::Display for InvalidPatternError {
@@ -37,8 +41,7 @@ impl fmt::Display for InvalidPatternError {
"found invalid UTF-8 in pattern at byte offset {} \ "found invalid UTF-8 in pattern at byte offset {} \
(use hex escape sequences to match arbitrary bytes \ (use hex escape sequences to match arbitrary bytes \
in a pattern, e.g., \\xFF): '{}'", in a pattern, e.g., \\xFF): '{}'",
self.valid_up_to, self.valid_up_to, self.original,
self.original,
) )
} }
} }
@@ -77,11 +80,9 @@ pub fn pattern_from_os(pattern: &OsStr) -> Result<&str, InvalidPatternError> {
pub fn pattern_from_bytes( pub fn pattern_from_bytes(
pattern: &[u8], pattern: &[u8],
) -> Result<&str, InvalidPatternError> { ) -> Result<&str, InvalidPatternError> {
str::from_utf8(pattern).map_err(|err| { str::from_utf8(pattern).map_err(|err| InvalidPatternError {
InvalidPatternError { original: escape(pattern),
original: escape(pattern), valid_up_to: err.valid_up_to(),
valid_up_to: err.valid_up_to(),
}
}) })
} }
@@ -117,10 +118,7 @@ pub fn patterns_from_stdin() -> io::Result<Vec<String>> {
let stdin = io::stdin(); let stdin = io::stdin();
let locked = stdin.lock(); let locked = stdin.lock();
patterns_from_reader(locked).map_err(|err| { patterns_from_reader(locked).map_err(|err| {
io::Error::new( io::Error::new(io::ErrorKind::Other, format!("<stdin>:{}", err))
io::ErrorKind::Other,
format!("<stdin>:{}", err),
)
}) })
} }
@@ -156,28 +154,20 @@ pub fn patterns_from_stdin() -> io::Result<Vec<String>> {
/// ``` /// ```
pub fn patterns_from_reader<R: io::Read>(rdr: R) -> io::Result<Vec<String>> { pub fn patterns_from_reader<R: io::Read>(rdr: R) -> io::Result<Vec<String>> {
let mut patterns = vec![]; let mut patterns = vec![];
let mut bufrdr = io::BufReader::new(rdr);
let mut line = vec![];
let mut line_number = 0; let mut line_number = 0;
while { io::BufReader::new(rdr).for_byte_line(|line| {
line.clear();
line_number += 1; line_number += 1;
bufrdr.read_until(b'\n', &mut line)? > 0 match pattern_from_bytes(line) {
} { Ok(pattern) => {
line.pop().unwrap(); // remove trailing '\n' patterns.push(pattern.to_string());
if line.last() == Some(&b'\r') { Ok(true)
line.pop().unwrap();
}
match pattern_from_bytes(&line) {
Ok(pattern) => patterns.push(pattern.to_string()),
Err(err) => {
return Err(io::Error::new(
io::ErrorKind::Other,
format!("{}: {}", line_number, err),
));
} }
Err(err) => Err(io::Error::new(
io::ErrorKind::Other,
format!("{}: {}", line_number, err),
)),
} }
} })?;
Ok(patterns) Ok(patterns)
} }
@@ -195,8 +185,8 @@ mod tests {
#[test] #[test]
#[cfg(unix)] #[cfg(unix)]
fn os() { fn os() {
use std::os::unix::ffi::OsStrExt;
use std::ffi::OsStr; use std::ffi::OsStr;
use std::os::unix::ffi::OsStrExt;
let pat = OsStr::from_bytes(b"abc\xFFxyz"); let pat = OsStr::from_bytes(b"abc\xFFxyz");
let err = pattern_from_os(pat).unwrap_err(); let err = pattern_from_os(pat).unwrap_err();

31
grep-cli/src/process.rs → crates/cli/src/process.rs
View File

@@ -33,7 +33,9 @@ impl CommandError {
} }
impl error::Error for CommandError { impl error::Error for CommandError {
fn description(&self) -> &str { "command error" } fn description(&self) -> &str {
"command error"
}
} }
impl fmt::Display for CommandError { impl fmt::Display for CommandError {
@@ -46,7 +48,12 @@ impl fmt::Display for CommandError {
write!(f, "<stderr is empty>") write!(f, "<stderr is empty>")
} else { } else {
let div = iter::repeat('-').take(79).collect::<String>(); let div = iter::repeat('-').take(79).collect::<String>();
write!(f, "\n{div}\n{msg}\n{div}", div=div, msg=msg.trim()) write!(
f,
"\n{div}\n{msg}\n{div}",
div = div,
msg = msg.trim()
)
} }
} }
} }
@@ -101,12 +108,11 @@ impl CommandReaderBuilder {
.stderr(process::Stdio::piped()) .stderr(process::Stdio::piped())
.spawn()?; .spawn()?;
let stdout = child.stdout.take().unwrap(); let stdout = child.stdout.take().unwrap();
let stderr = let stderr = if self.async_stderr {
if self.async_stderr { StderrReader::async(child.stderr.take().unwrap())
StderrReader::async(child.stderr.take().unwrap()) } else {
} else { StderrReader::sync(child.stderr.take().unwrap())
StderrReader::sync(child.stderr.take().unwrap()) };
};
Ok(CommandReader { Ok(CommandReader {
child: child, child: child,
stdout: stdout, stdout: stdout,
@@ -226,9 +232,8 @@ enum StderrReader {
impl StderrReader { impl StderrReader {
/// Create a reader for stderr that reads contents asynchronously. /// Create a reader for stderr that reads contents asynchronously.
fn async(mut stderr: process::ChildStderr) -> StderrReader { fn async(mut stderr: process::ChildStderr) -> StderrReader {
let handle = thread::spawn(move || { let handle =
stderr_to_command_error(&mut stderr) thread::spawn(move || stderr_to_command_error(&mut stderr));
});
StderrReader::Async(Some(handle)) StderrReader::Async(Some(handle))
} }
@@ -247,9 +252,7 @@ impl StderrReader {
let handle = handle let handle = handle
.take() .take()
.expect("read_to_end cannot be called more than once"); .expect("read_to_end cannot be called more than once");
handle handle.join().expect("stderr reading thread does not panic")
.join()
.expect("stderr reading thread does not panic")
} }
StderrReader::Sync(ref mut stderr) => { StderrReader::Sync(ref mut stderr) => {
stderr_to_command_error(stderr) stderr_to_command_error(stderr)

0
grep-cli/src/wtr.rs → crates/cli/src/wtr.rs
View File

15
crates/core/README.md Normal file
View File

@@ -0,0 +1,15 @@
ripgrep core
------------
This is the core ripgrep crate. In particular, `main.rs` is where the `main`
function lives.
Most of ripgrep core consists of two things:
* The definition of the CLI interface, including docs for every flag.
* Glue code that brings the `grep-matcher`, `grep-regex`, `grep-searcher` and
`grep-printer` crates together to actually execute the search.
Currently, there are no plans to make ripgrep core available as an independent
library. However, much of the heavy lifting of ripgrep is done via its
constituent crates, which can be reused independent of ripgrep. Unfortunately,
there is no guide or tutorial to teach folks how to do this yet.

1441
src/app.rs → crates/core/app.rs
View File

File diff suppressed because it is too large Load Diff

467
src/args.rs → crates/core/args.rs
View File

@@ -17,11 +17,8 @@ use grep::pcre2::{
RegexMatcherBuilder as PCRE2RegexMatcherBuilder, RegexMatcherBuilder as PCRE2RegexMatcherBuilder,
}; };
use grep::printer::{ use grep::printer::{
ColorSpecs, Stats, default_color_specs, ColorSpecs, JSONBuilder, Standard, StandardBuilder,
JSON, JSONBuilder, Stats, Summary, SummaryBuilder, SummaryKind, JSON,
Standard, StandardBuilder,
Summary, SummaryBuilder, SummaryKind,
default_color_specs,
}; };
use grep::regex::{ use grep::regex::{
RegexMatcher as RustRegexMatcher, RegexMatcher as RustRegexMatcher,
@@ -36,15 +33,12 @@ use ignore::{Walk, WalkBuilder, WalkParallel};
use log; use log;
use num_cpus; use num_cpus;
use regex; use regex;
use termcolor::{ use termcolor::{BufferWriter, ColorChoice, WriteColor};
WriteColor,
BufferWriter, ColorChoice,
};
use crate::app; use crate::app;
use crate::config; use crate::config;
use crate::logger::Logger; use crate::logger::Logger;
use crate::messages::{set_messages, set_ignore_messages}; use crate::messages::{set_ignore_messages, set_messages};
use crate::path_printer::{PathPrinter, PathPrinterBuilder}; use crate::path_printer::{PathPrinter, PathPrinterBuilder};
use crate::search::{ use crate::search::{
PatternMatcher, Printer, SearchWorker, SearchWorkerBuilder, PatternMatcher, Printer, SearchWorker, SearchWorkerBuilder,
@@ -84,11 +78,9 @@ impl Command {
match *self { match *self {
Search | SearchParallel => true, Search | SearchParallel => true,
| SearchNever SearchNever | Files | FilesParallel | Types | PCRE2Version => {
| Files false
| FilesParallel }
| Types
| PCRE2Version => false,
} }
} }
} }
@@ -210,15 +202,12 @@ impl Args {
.printer_standard(self.paths(), wtr, separator_search) .printer_standard(self.paths(), wtr, separator_search)
.map(Printer::Standard) .map(Printer::Standard)
} }
OutputKind::Summary => { OutputKind::Summary => self
self.matches() .matches()
.printer_summary(self.paths(), wtr) .printer_summary(self.paths(), wtr)
.map(Printer::Summary) .map(Printer::Summary),
}
OutputKind::JSON => { OutputKind::JSON => {
self.matches() self.matches().printer_json(wtr).map(Printer::JSON)
.printer_json(wtr)
.map(Printer::JSON)
} }
} }
} }
@@ -405,11 +394,11 @@ enum SortByKind {
impl SortBy { impl SortBy {
fn asc(kind: SortByKind) -> SortBy { fn asc(kind: SortByKind) -> SortBy {
SortBy { reverse: false, kind: kind } SortBy { reverse: false, kind }
} }
fn desc(kind: SortByKind) -> SortBy { fn desc(kind: SortByKind) -> SortBy {
SortBy { reverse: true, kind: kind } SortBy { reverse: true, kind }
} }
fn none() -> SortBy { fn none() -> SortBy {
@@ -452,29 +441,23 @@ impl SortBy {
} }
SortByKind::LastModified => { SortByKind::LastModified => {
builder.sort_by_file_path(move |a, b| { builder.sort_by_file_path(move |a, b| {
sort_by_metadata_time( sort_by_metadata_time(a, b, self.reverse, |md| {
a, b, md.modified()
self.reverse, })
|md| md.modified(),
)
}); });
} }
SortByKind::LastAccessed => { SortByKind::LastAccessed => {
builder.sort_by_file_path(move |a, b| { builder.sort_by_file_path(move |a, b| {
sort_by_metadata_time( sort_by_metadata_time(a, b, self.reverse, |md| {
a, b, md.accessed()
self.reverse, })
|md| md.accessed(),
)
}); });
} }
SortByKind::Created => { SortByKind::Created => {
builder.sort_by_file_path(move |a, b| { builder.sort_by_file_path(move |a, b| {
sort_by_metadata_time( sort_by_metadata_time(a, b, self.reverse, |md| {
a, b, md.created()
self.reverse, })
|md| md.created(),
)
}); });
} }
} }
@@ -520,7 +503,7 @@ impl EncodingMode {
fn has_explicit_encoding(&self) -> bool { fn has_explicit_encoding(&self) -> bool {
match self { match self {
EncodingMode::Some(_) => true, EncodingMode::Some(_) => true,
_ => false _ => false,
} }
} }
} }
@@ -568,19 +551,18 @@ impl ArgMatches {
let patterns = self.patterns()?; let patterns = self.patterns()?;
let matcher = self.matcher(&patterns)?; let matcher = self.matcher(&patterns)?;
let mut paths = self.paths(); let mut paths = self.paths();
let using_default_path = let using_default_path = if paths.is_empty() {
if paths.is_empty() { paths.push(self.path_default());
paths.push(self.path_default()); true
true } else {
} else { false
false };
};
Ok(Args(Arc::new(ArgsImp { Ok(Args(Arc::new(ArgsImp {
matches: self, matches: self,
patterns: patterns, patterns,
matcher: matcher, matcher,
paths: paths, paths,
using_default_path: using_default_path, using_default_path,
}))) })))
} }
} }
@@ -594,54 +576,78 @@ impl ArgMatches {
/// ///
/// If there was a problem building the matcher (e.g., a syntax error), /// If there was a problem building the matcher (e.g., a syntax error),
/// then this returns an error. /// then this returns an error.
#[cfg(feature = "pcre2")]
fn matcher(&self, patterns: &[String]) -> Result<PatternMatcher> { fn matcher(&self, patterns: &[String]) -> Result<PatternMatcher> {
if self.is_present("pcre2") { if self.is_present("pcre2") {
let matcher = self.matcher_pcre2(patterns)?; self.matcher_engine("pcre2", patterns)
Ok(PatternMatcher::PCRE2(matcher))
} else if self.is_present("auto-hybrid-regex") { } else if self.is_present("auto-hybrid-regex") {
let rust_err = match self.matcher_rust(patterns) { self.matcher_engine("auto", patterns)
Ok(matcher) => return Ok(PatternMatcher::RustRegex(matcher)),
Err(err) => err,
};
log::debug!(
"error building Rust regex in hybrid mode:\n{}", rust_err,
);
let pcre_err = match self.matcher_pcre2(patterns) {
Ok(matcher) => return Ok(PatternMatcher::PCRE2(matcher)),
Err(err) => err,
};
Err(From::from(format!(
"regex could not be compiled with either the default regex \
engine or with PCRE2.\n\n\
default regex engine error:\n{}\n{}\n{}\n\n\
PCRE2 regex engine error:\n{}",
"~".repeat(79), rust_err, "~".repeat(79), pcre_err,
)))
} else { } else {
let matcher = match self.matcher_rust(patterns) { let engine = self.value_of_lossy("engine").unwrap();
Ok(matcher) => matcher, self.matcher_engine(&engine, patterns)
Err(err) => {
return Err(From::from(suggest_pcre2(err.to_string())));
}
};
Ok(PatternMatcher::RustRegex(matcher))
} }
} }
/// Return the matcher that should be used for searching. /// Return the matcher that should be used for searching using engine
/// as the engine for the patterns.
/// ///
/// If there was a problem building the matcher (e.g., a syntax error), /// If there was a problem building the matcher (e.g., a syntax error),
/// then this returns an error. /// then this returns an error.
#[cfg(not(feature = "pcre2"))] fn matcher_engine(
fn matcher(&self, patterns: &[String]) -> Result<PatternMatcher> { &self,
if self.is_present("pcre2") { engine: &str,
return Err(From::from( patterns: &[String],
) -> Result<PatternMatcher> {
match engine {
"default" => {
let matcher = match self.matcher_rust(patterns) {
Ok(matcher) => matcher,
Err(err) => {
return Err(From::from(suggest(err.to_string())));
}
};
Ok(PatternMatcher::RustRegex(matcher))
}
#[cfg(feature = "pcre2")]
"pcre2" => {
let matcher = self.matcher_pcre2(patterns)?;
Ok(PatternMatcher::PCRE2(matcher))
}
#[cfg(not(feature = "pcre2"))]
"pcre2" => Err(From::from(
"PCRE2 is not available in this build of ripgrep", "PCRE2 is not available in this build of ripgrep",
)); )),
"auto" => {
let rust_err = match self.matcher_rust(patterns) {
Ok(matcher) => {
return Ok(PatternMatcher::RustRegex(matcher));
}
Err(err) => err,
};
log::debug!(
"error building Rust regex in hybrid mode:\n{}",
rust_err,
);
let pcre_err = match self.matcher_engine("pcre2", patterns) {
Ok(matcher) => return Ok(matcher),
Err(err) => err,
};
Err(From::from(format!(
"regex could not be compiled with either the default \
regex engine or with PCRE2.\n\n\
default regex engine error:\n{}\n{}\n{}\n\n\
PCRE2 regex engine error:\n{}",
"~".repeat(79),
rust_err,
"~".repeat(79),
pcre_err,
)))
}
_ => Err(From::from(format!(
"unrecognized regex engine '{}'",
engine
))),
} }
let matcher = self.matcher_rust(patterns)?;
Ok(PatternMatcher::RustRegex(matcher))
} }
/// Build a matcher using Rust's regex engine. /// Build a matcher using Rust's regex engine.
@@ -654,20 +660,16 @@ impl ArgMatches {
.case_smart(self.case_smart()) .case_smart(self.case_smart())
.case_insensitive(self.case_insensitive()) .case_insensitive(self.case_insensitive())
.multi_line(true) .multi_line(true)
.unicode(true) .unicode(self.unicode())
.octal(false) .octal(false)
.word(self.is_present("word-regexp")); .word(self.is_present("word-regexp"));
if self.is_present("multiline") { if self.is_present("multiline") {
builder.dot_matches_new_line(self.is_present("multiline-dotall")); builder.dot_matches_new_line(self.is_present("multiline-dotall"));
if self.is_present("crlf") { if self.is_present("crlf") {
builder builder.crlf(true).line_terminator(None);
.crlf(true)
.line_terminator(None);
} }
} else { } else {
builder builder.line_terminator(Some(b'\n')).dot_matches_new_line(false);
.line_terminator(Some(b'\n'))
.dot_matches_new_line(false);
if self.is_present("crlf") { if self.is_present("crlf") {
builder.crlf(true); builder.crlf(true);
} }
@@ -686,12 +688,11 @@ impl ArgMatches {
if let Some(limit) = self.dfa_size_limit()? { if let Some(limit) = self.dfa_size_limit()? {
builder.dfa_size_limit(limit); builder.dfa_size_limit(limit);
} }
let res = let res = if self.is_present("fixed-strings") {
if self.is_present("fixed-strings") { builder.build_literals(patterns)
builder.build_literals(patterns) } else {
} else { builder.build(&patterns.join("|"))
builder.build(&patterns.join("|")) };
};
match res { match res {
Ok(m) => Ok(m), Ok(m) => Ok(m),
Err(err) => Err(From::from(suggest_multiline(err.to_string()))), Err(err) => Err(From::from(suggest_multiline(err.to_string()))),
@@ -718,9 +719,9 @@ impl ArgMatches {
// The PCRE2 docs say that 32KB is the default, and that 1MB // The PCRE2 docs say that 32KB is the default, and that 1MB
// should be big enough for anything. But let's crank it to // should be big enough for anything. But let's crank it to
// 10MB. // 10MB.
.max_jit_stack_size(Some(10 * (1<<20))); .max_jit_stack_size(Some(10 * (1 << 20)));
} }
if self.pcre2_unicode() { if self.unicode() {
builder.utf(true).ucp(true); builder.utf(true).ucp(true);
if self.encoding()?.has_explicit_encoding() { if self.encoding()?.has_explicit_encoding() {
// SAFETY: If an encoding was specified, then we're guaranteed // SAFETY: If an encoding was specified, then we're guaranteed
@@ -784,7 +785,7 @@ impl ArgMatches {
.byte_offset(self.is_present("byte-offset")) .byte_offset(self.is_present("byte-offset"))
.trim_ascii(self.is_present("trim")) .trim_ascii(self.is_present("trim"))
.separator_search(None) .separator_search(None)
.separator_context(Some(self.context_separator())) .separator_context(self.context_separator())
.separator_field_match(b":".to_vec()) .separator_field_match(b":".to_vec())
.separator_field_context(b"-".to_vec()) .separator_field_context(b"-".to_vec())
.separator_path(self.path_separator()?) .separator_path(self.path_separator()?)
@@ -812,6 +813,7 @@ impl ArgMatches {
.stats(self.stats()) .stats(self.stats())
.path(self.with_filename(paths)) .path(self.with_filename(paths))
.max_matches(self.max_count()?) .max_matches(self.max_count()?)
.exclude_zero(!self.is_present("include-zero"))
.separator_field(b":".to_vec()) .separator_field(b":".to_vec())
.separator_path(self.path_separator()?) .separator_path(self.path_separator()?)
.path_terminator(self.path_terminator()); .path_terminator(self.path_terminator());
@@ -821,14 +823,13 @@ impl ArgMatches {
/// Build a searcher from the command line parameters. /// Build a searcher from the command line parameters.
fn searcher(&self, paths: &[PathBuf]) -> Result<Searcher> { fn searcher(&self, paths: &[PathBuf]) -> Result<Searcher> {
let (ctx_before, ctx_after) = self.contexts()?; let (ctx_before, ctx_after) = self.contexts()?;
let line_term = let line_term = if self.is_present("crlf") {
if self.is_present("crlf") { LineTerminator::crlf()
LineTerminator::crlf() } else if self.is_present("null-data") {
} else if self.is_present("null-data") { LineTerminator::byte(b'\x00')
LineTerminator::byte(b'\x00') } else {
} else { LineTerminator::byte(b'\n')
LineTerminator::byte(b'\n') };
};
let mut builder = SearcherBuilder::new(); let mut builder = SearcherBuilder::new();
builder builder
.line_terminator(line_term) .line_terminator(line_term)
@@ -861,9 +862,11 @@ impl ArgMatches {
for path in &paths[1..] { for path in &paths[1..] {
builder.add(path); builder.add(path);
} }
for path in self.ignore_paths() { if !self.no_ignore_files() {
if let Some(err) = builder.add_ignore(path) { for path in self.ignore_paths() {
ignore_message!("{}", err); if let Some(err) = builder.add_ignore(path) {
ignore_message!("{}", err);
}
} }
} }
builder builder
@@ -880,7 +883,8 @@ impl ArgMatches {
.ignore(!self.no_ignore_dot()) .ignore(!self.no_ignore_dot())
.git_global(!self.no_ignore_vcs() && !self.no_ignore_global()) .git_global(!self.no_ignore_vcs() && !self.no_ignore_global())
.git_ignore(!self.no_ignore_vcs()) .git_ignore(!self.no_ignore_vcs())
.git_exclude(!self.no_ignore_vcs()) .git_exclude(!self.no_ignore_vcs() && !self.no_ignore_exclude())
.require_git(!self.is_present("no-require-git"))
.ignore_case_insensitive(self.ignore_file_case_insensitive()); .ignore_case_insensitive(self.ignore_file_case_insensitive());
if !self.no_ignore() { if !self.no_ignore() {
builder.add_custom_ignore_filename(".rgignore"); builder.add_custom_ignore_filename(".rgignore");
@@ -900,12 +904,9 @@ impl ArgMatches {
/// Returns the form of binary detection to perform on files that are /// Returns the form of binary detection to perform on files that are
/// implicitly searched via recursive directory traversal. /// implicitly searched via recursive directory traversal.
fn binary_detection_implicit(&self) -> BinaryDetection { fn binary_detection_implicit(&self) -> BinaryDetection {
let none = let none = self.is_present("text") || self.is_present("null-data");
self.is_present("text")
|| self.is_present("null-data");
let convert = let convert =
self.is_present("binary") self.is_present("binary") || self.unrestricted_count() >= 3;
|| self.unrestricted_count() >= 3;
if none { if none {
BinaryDetection::none() BinaryDetection::none()
} else if convert { } else if convert {
@@ -923,9 +924,7 @@ impl ArgMatches {
/// as a filter (but quitting immediately once a NUL byte is seen), and we /// as a filter (but quitting immediately once a NUL byte is seen), and we
/// should never filter out files that the user wants to explicitly search. /// should never filter out files that the user wants to explicitly search.
fn binary_detection_explicit(&self) -> BinaryDetection { fn binary_detection_explicit(&self) -> BinaryDetection {
let none = let none = self.is_present("text") || self.is_present("null-data");
self.is_present("text")
|| self.is_present("null-data");
if none { if none {
BinaryDetection::none() BinaryDetection::none()
} else { } else {
@@ -953,8 +952,8 @@ impl ArgMatches {
/// case is disabled. /// case is disabled.
fn case_smart(&self) -> bool { fn case_smart(&self) -> bool {
self.is_present("smart-case") self.is_present("smart-case")
&& !self.is_present("ignore-case") && !self.is_present("ignore-case")
&& !self.is_present("case-sensitive") && !self.is_present("case-sensitive")
} }
/// Returns the user's color choice based on command line parameters and /// Returns the user's color choice based on command line parameters and
@@ -1010,20 +1009,20 @@ impl ArgMatches {
let after = self.usize_of("after-context")?.unwrap_or(0); let after = self.usize_of("after-context")?.unwrap_or(0);
let before = self.usize_of("before-context")?.unwrap_or(0); let before = self.usize_of("before-context")?.unwrap_or(0);
let both = self.usize_of("context")?.unwrap_or(0); let both = self.usize_of("context")?.unwrap_or(0);
Ok(if both > 0 { Ok(if both > 0 { (both, both) } else { (before, after) })
(both, both)
} else {
(before, after)
})
} }
/// Returns the unescaped context separator in UTF-8 bytes. /// Returns the unescaped context separator in UTF-8 bytes.
/// ///
/// If one was not provided, the default `--` is returned. /// If one was not provided, the default `--` is returned.
fn context_separator(&self) -> Vec<u8> { /// If --no-context-separator is passed, None is returned.
match self.value_of_os("context-separator") { fn context_separator(&self) -> Option<Vec<u8>> {
None => b"--".to_vec(), let nosep = self.is_present("no-context-separator");
Some(sep) => cli::unescape_os(&sep), let sep = self.value_of_os("context-separator");
match (nosep, sep) {
(true, _) => None,
(false, None) => Some(b"--".to_vec()),
(false, Some(sep)) => Some(cli::unescape_os(&sep)),
} }
} }
@@ -1092,7 +1091,7 @@ impl ArgMatches {
Ok(if self.heading() { Ok(if self.heading() {
Some(b"".to_vec()) Some(b"".to_vec())
} else if ctx_before > 0 || ctx_after > 0 { } else if ctx_before > 0 || ctx_after > 0 {
Some(self.context_separator().clone()) self.context_separator()
} else { } else {
None None
}) })
@@ -1105,8 +1104,8 @@ impl ArgMatches {
false false
} else { } else {
cli::is_tty_stdout() cli::is_tty_stdout()
|| self.is_present("heading") || self.is_present("heading")
|| self.is_present("pretty") || self.is_present("pretty")
} }
} }
@@ -1162,10 +1161,10 @@ impl ArgMatches {
// tty for human consumption, except for one interesting case: when // tty for human consumption, except for one interesting case: when
// we're only searching stdin. This makes pipelines work as expected. // we're only searching stdin. This makes pipelines work as expected.
(cli::is_tty_stdout() && !self.is_only_stdin(paths)) (cli::is_tty_stdout() && !self.is_only_stdin(paths))
|| self.is_present("line-number") || self.is_present("line-number")
|| self.is_present("column") || self.is_present("column")
|| self.is_present("pretty") || self.is_present("pretty")
|| self.is_present("vimgrep") || self.is_present("vimgrep")
} }
/// The maximum number of columns allowed on each line. /// The maximum number of columns allowed on each line.
@@ -1226,6 +1225,19 @@ impl ArgMatches {
self.is_present("no-ignore-dot") || self.no_ignore() self.is_present("no-ignore-dot") || self.no_ignore()
} }
/// Returns true if local exclude (ignore) files should be ignored.
fn no_ignore_exclude(&self) -> bool {
self.is_present("no-ignore-exclude") || self.no_ignore()
}
/// Returns true if explicitly given ignore files should be ignored.
fn no_ignore_files(&self) -> bool {
// We don't look at no-ignore here because --no-ignore is explicitly
// documented to not override --ignore-file. We could change this, but
// it would be a fairly severe breaking change.
self.is_present("no-ignore-files")
}
/// Returns true if global ignore files should be ignored. /// Returns true if global ignore files should be ignored.
fn no_ignore_global(&self) -> bool { fn no_ignore_global(&self) -> bool {
self.is_present("no-ignore-global") || self.no_ignore() self.is_present("no-ignore-global") || self.no_ignore()
@@ -1253,8 +1265,7 @@ impl ArgMatches {
} }
let (count, count_matches) = self.counts(); let (count, count_matches) = self.counts();
let summary = let summary = count
count
|| count_matches || count_matches
|| self.is_present("files-with-matches") || self.is_present("files-with-matches")
|| self.is_present("files-without-match"); || self.is_present("files-without-match");
@@ -1267,13 +1278,23 @@ impl ArgMatches {
/// Builds the set of glob overrides from the command line flags. /// Builds the set of glob overrides from the command line flags.
fn overrides(&self) -> Result<Override> { fn overrides(&self) -> Result<Override> {
let mut builder = OverrideBuilder::new(env::current_dir()?); let globs = self.values_of_lossy_vec("glob");
for glob in self.values_of_lossy_vec("glob") { let iglobs = self.values_of_lossy_vec("iglob");
if globs.is_empty() && iglobs.is_empty() {
return Ok(Override::empty());
}
let mut builder = OverrideBuilder::new(current_dir()?);
// Make all globs case insensitive with --glob-case-insensitive.
if self.is_present("glob-case-insensitive") {
builder.case_insensitive(true).unwrap();
}
for glob in globs {
builder.add(&glob)?; builder.add(&glob)?;
} }
// This only enables case insensitivity for subsequent globs. // This only enables case insensitivity for subsequent globs.
builder.case_insensitive(true).unwrap(); builder.case_insensitive(true).unwrap();
for glob in self.values_of_lossy_vec("iglob") { for glob in iglobs {
builder.add(&glob)?; builder.add(&glob)?;
} }
Ok(builder.build()?) Ok(builder.build()?)
@@ -1304,10 +1325,10 @@ impl ArgMatches {
/// be used when ripgrep is not otherwise given at least one file path /// be used when ripgrep is not otherwise given at least one file path
/// as a positional argument. /// as a positional argument.
fn path_default(&self) -> PathBuf { fn path_default(&self) -> PathBuf {
let file_is_stdin = self.values_of_os("file") let file_is_stdin = self
.values_of_os("file")
.map_or(false, |mut files| files.any(|f| f == "-")); .map_or(false, |mut files| files.any(|f| f == "-"));
let search_cwd = let search_cwd = !cli::is_readable_stdin()
!cli::is_readable_stdin()
|| (self.is_present("file") && file_is_stdin) || (self.is_present("file") && file_is_stdin)
|| self.is_present("files") || self.is_present("files")
|| self.is_present("type-list") || self.is_present("type-list")
@@ -1336,8 +1357,8 @@ impl ArgMatches {
the given separator is {} bytes: {}\n\ the given separator is {} bytes: {}\n\
In some shells on Windows '/' is automatically \ In some shells on Windows '/' is automatically \
expanded. Use '//' instead.", expanded. Use '//' instead.",
sep.len(), sep.len(),
cli::escape(&sep), cli::escape(&sep),
))) )))
} else { } else {
Ok(Some(sep[0])) Ok(Some(sep[0]))
@@ -1387,14 +1408,16 @@ impl ArgMatches {
if let Some(paths) = self.values_of_os("file") { if let Some(paths) = self.values_of_os("file") {
for path in paths { for path in paths {
if path == "-" { if path == "-" {
pats.extend(cli::patterns_from_stdin()? pats.extend(
.into_iter() cli::patterns_from_stdin()?
.map(|p| self.pattern_from_string(p)) .into_iter()
.map(|p| self.pattern_from_string(p)),
); );
} else { } else {
pats.extend(cli::patterns_from_path(path)? pats.extend(
.into_iter() cli::patterns_from_path(path)?
.map(|p| self.pattern_from_string(p)) .into_iter()
.map(|p| self.pattern_from_string(p)),
); );
} }
} }
@@ -1475,8 +1498,12 @@ impl ArgMatches {
/// flag. If no --pre-globs are available, then this always returns an /// flag. If no --pre-globs are available, then this always returns an
/// empty set of globs. /// empty set of globs.
fn preprocessor_globs(&self) -> Result<Override> { fn preprocessor_globs(&self) -> Result<Override> {
let mut builder = OverrideBuilder::new(env::current_dir()?); let globs = self.values_of_lossy_vec("pre-glob");
for glob in self.values_of_lossy_vec("pre-glob") { if globs.is_empty() {
return Ok(Override::empty());
}
let mut builder = OverrideBuilder::new(current_dir()?);
for glob in globs {
builder.add(&glob)?; builder.add(&glob)?;
} }
Ok(builder.build()?) Ok(builder.build()?)
@@ -1503,7 +1530,7 @@ impl ArgMatches {
None => match self.value_of_lossy("sortr") { None => match self.value_of_lossy("sortr") {
None => return Ok(SortBy::none()), None => return Ok(SortBy::none()),
Some(choice) => SortBy::desc(SortByKind::new(&choice)), Some(choice) => SortBy::desc(SortByKind::new(&choice)),
} },
Some(choice) => SortBy::asc(SortByKind::new(&choice)), Some(choice) => SortBy::asc(SortByKind::new(&choice)),
}; };
Ok(sortby) Ok(sortby)
@@ -1546,11 +1573,7 @@ impl ArgMatches {
return Ok(1); return Ok(1);
} }
let threads = self.usize_of("threads")?.unwrap_or(0); let threads = self.usize_of("threads")?.unwrap_or(0);
Ok(if threads == 0 { Ok(if threads == 0 { cmp::min(12, num_cpus::get()) } else { threads })
cmp::min(12, num_cpus::get())
} else {
threads
})
} }
/// Builds a file type matcher from the command line flags. /// Builds a file type matcher from the command line flags.
@@ -1577,11 +1600,17 @@ impl ArgMatches {
self.occurrences_of("unrestricted") self.occurrences_of("unrestricted")
} }
/// Returns true if and only if PCRE2's Unicode mode should be enabled. /// Returns true if and only if Unicode mode should be enabled.
fn unicode(&self) -> bool {
// Unicode mode is enabled by default, so only disable it when
// --no-unicode is given explicitly.
!(self.is_present("no-unicode") || self.is_present("no-pcre2-unicode"))
}
/// Returns true if and only if PCRE2 is enabled and its Unicode mode is
/// enabled.
fn pcre2_unicode(&self) -> bool { fn pcre2_unicode(&self) -> bool {
// PCRE2 Unicode is enabled by default, so only disable it when told self.is_present("pcre2") && self.unicode()
// to do so explicitly.
self.is_present("pcre2") && !self.is_present("no-pcre2-unicode")
} }
/// Returns true if and only if file names containing each match should /// Returns true if and only if file names containing each match should
@@ -1590,10 +1619,13 @@ impl ArgMatches {
if self.is_present("no-filename") { if self.is_present("no-filename") {
false false
} else { } else {
let path_stdin = Path::new("-");
self.is_present("with-filename") self.is_present("with-filename")
|| self.is_present("vimgrep") || self.is_present("vimgrep")
|| paths.len() > 1 || paths.len() > 1
|| paths.get(0).map_or(false, |p| p.is_dir()) || paths
.get(0)
.map_or(false, |p| p != path_stdin && p.is_dir())
} }
} }
} }
@@ -1616,11 +1648,7 @@ impl ArgMatches {
None => return Ok(None), None => return Ok(None),
Some(n) => n, Some(n) => n,
}; };
Ok(if n == 0 { Ok(if n == 0 { None } else { Some(n) })
None
} else {
Some(n)
})
} }
/// Safely reads an arg value with the given name, and if it's present, /// Safely reads an arg value with the given name, and if it's present,
@@ -1678,27 +1706,52 @@ impl ArgMatches {
} }
} }
/// Inspect an error resulting from building a Rust regex matcher, and if it's
/// believed to correspond to a syntax error that another engine could handle,
/// then add a message to suggest the use of the engine flag.
fn suggest(msg: String) -> String {
if let Some(pcre_msg) = suggest_pcre2(&msg) {
return pcre_msg;
}
msg
}
/// Inspect an error resulting from building a Rust regex matcher, and if it's /// Inspect an error resulting from building a Rust regex matcher, and if it's
/// believed to correspond to a syntax error that PCRE2 could handle, then /// believed to correspond to a syntax error that PCRE2 could handle, then
/// add a message to suggest the use of -P/--pcre2. /// add a message to suggest the use of -P/--pcre2.
#[cfg(feature = "pcre2")] fn suggest_pcre2(msg: &str) -> Option<String> {
fn suggest_pcre2(msg: String) -> String { #[cfg(feature = "pcre2")]
if !msg.contains("backreferences") && !msg.contains("look-around") { fn suggest(msg: &str) -> Option<String> {
msg if !msg.contains("backreferences") && !msg.contains("look-around") {
} else { None
format!("{} } else {
Some(format!(
"{}
Consider enabling PCRE2 with the --pcre2 flag, which can handle backreferences Consider enabling PCRE2 with the --pcre2 flag, which can handle backreferences
and look-around.", msg) and look-around.",
msg
))
}
} }
#[cfg(not(feature = "pcre2"))]
fn suggest(_: &str) -> Option<String> {
None
}
suggest(msg)
} }
fn suggest_multiline(msg: String) -> String { fn suggest_multiline(msg: String) -> String {
if msg.contains("the literal") && msg.contains("not allowed") { if msg.contains("the literal") && msg.contains("not allowed") {
format!("{} format!(
"{}
Consider enabling multiline mode with the --multiline flag (or -U for short). Consider enabling multiline mode with the --multiline flag (or -U for short).
When multiline mode is enabled, new line characters can be matched.", msg) When multiline mode is enabled, new line characters can be matched.",
msg
)
} else { } else {
msg msg
} }
@@ -1706,10 +1759,7 @@ When multiline mode is enabled, new line characters can be matched.", msg)
/// Convert the result of parsing a human readable file size to a `usize`, /// Convert the result of parsing a human readable file size to a `usize`,
/// failing if the type does not fit. /// failing if the type does not fit.
fn u64_to_usize( fn u64_to_usize(arg_name: &str, value: Option<u64>) -> Result<Option<usize>> {
arg_name: &str,
value: Option<u64>,
) -> Result<Option<usize>> {
use std::usize; use std::usize;
let value = match value { let value = match value {
@@ -1734,7 +1784,8 @@ fn sort_by_metadata_time<G>(
reverse: bool, reverse: bool,
get_time: G, get_time: G,
) -> cmp::Ordering ) -> cmp::Ordering
where G: Fn(&fs::Metadata) -> io::Result<SystemTime> where
G: Fn(&fs::Metadata) -> io::Result<SystemTime>,
{ {
let t1 = match p1.metadata().and_then(|md| get_time(&md)) { let t1 = match p1.metadata().and_then(|md| get_time(&md)) {
Ok(t) => t, Ok(t) => t,
@@ -1757,11 +1808,10 @@ where G: Fn(&fs::Metadata) -> io::Result<SystemTime>
/// corresponds to a `--help` or `--version` request. In which case, the /// corresponds to a `--help` or `--version` request. In which case, the
/// corresponding output is printed and the current process is exited /// corresponding output is printed and the current process is exited
/// successfully. /// successfully.
fn clap_matches<I, T>( fn clap_matches<I, T>(args: I) -> Result<clap::ArgMatches<'static>>
args: I, where
) -> Result<clap::ArgMatches<'static>> I: IntoIterator<Item = T>,
where I: IntoIterator<Item=T>, T: Into<OsString> + Clone,
T: Into<OsString> + Clone
{ {
let err = match app::app().get_matches_from_safe(args) { let err = match app::app().get_matches_from_safe(args) {
Ok(matches) => return Ok(matches), Ok(matches) => return Ok(matches),
@@ -1779,3 +1829,26 @@ where I: IntoIterator<Item=T>,
let _ = write!(io::stdout(), "{}", err); let _ = write!(io::stdout(), "{}", err);
process::exit(0); process::exit(0);
} }
/// Attempts to discover the current working directory. This mostly just defers
/// to the standard library, however, such things will fail if ripgrep is in
/// a directory that no longer exists. We attempt some fallback mechanisms,
/// such as querying the PWD environment variable, but otherwise return an
/// error.
fn current_dir() -> Result<PathBuf> {
let err = match env::current_dir() {
Err(err) => err,
Ok(cwd) => return Ok(cwd),
};
if let Some(cwd) = env::var_os("PWD") {
if !cwd.is_empty() {
return Ok(PathBuf::from(cwd));
}
}
Err(format!(
"failed to get current working directory: {} \
--- did your CWD get deleted?",
err,
)
.into())
}

53
src/config.rs → crates/core/config.rs
View File

@@ -4,12 +4,12 @@
use std::env; use std::env;
use std::error::Error; use std::error::Error;
use std::ffi::OsString;
use std::fs::File; use std::fs::File;
use std::io; use std::io;
use std::ffi::OsString;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use bstr::io::BufReadExt; use bstr::{io::BufReadExt, ByteSlice};
use log; use log;
use crate::Result; use crate::Result;
@@ -55,7 +55,7 @@ pub fn args() -> Vec<OsString> {
/// for each line in addition to successfully parsed arguments. /// for each line in addition to successfully parsed arguments.
fn parse<P: AsRef<Path>>( fn parse<P: AsRef<Path>>(
path: P, path: P,
) -> Result<(Vec<OsString>, Vec<Box<Error>>)> { ) -> Result<(Vec<OsString>, Vec<Box<dyn Error>>)> {
let path = path.as_ref(); let path = path.as_ref();
match File::open(&path) { match File::open(&path) {
Ok(file) => parse_reader(file), Ok(file) => parse_reader(file),
@@ -76,7 +76,7 @@ fn parse<P: AsRef<Path>>(
/// in addition to successfully parsed arguments. /// in addition to successfully parsed arguments.
fn parse_reader<R: io::Read>( fn parse_reader<R: io::Read>(
rdr: R, rdr: R,
) -> Result<(Vec<OsString>, Vec<Box<Error>>)> { ) -> Result<(Vec<OsString>, Vec<Box<dyn Error>>)> {
let bufrdr = io::BufReader::new(rdr); let bufrdr = io::BufReader::new(rdr);
let (mut args, mut errs) = (vec![], vec![]); let (mut args, mut errs) = (vec![], vec![]);
let mut line_number = 0; let mut line_number = 0;
@@ -102,12 +102,13 @@ fn parse_reader<R: io::Read>(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::ffi::OsString;
use super::parse_reader; use super::parse_reader;
use std::ffi::OsString;
#[test] #[test]
fn basic() { fn basic() {
let (args, errs) = parse_reader(&b"\ let (args, errs) = parse_reader(
&b"\
# Test # Test
--context=0 --context=0
--smart-case --smart-case
@@ -116,13 +117,13 @@ mod tests {
# --bar # --bar
--foo --foo
"[..]).unwrap(); "[..],
)
.unwrap();
assert!(errs.is_empty()); assert!(errs.is_empty());
let args: Vec<String> = let args: Vec<String> =
args.into_iter().map(|s| s.into_string().unwrap()).collect(); args.into_iter().map(|s| s.into_string().unwrap()).collect();
assert_eq!(args, vec![ assert_eq!(args, vec!["--context=0", "--smart-case", "-u", "--foo",]);
"--context=0", "--smart-case", "-u", "--foo",
]);
} }
// We test that we can handle invalid UTF-8 on Unix-like systems. // We test that we can handle invalid UTF-8 on Unix-like systems.
@@ -131,32 +132,38 @@ mod tests {
fn error() { fn error() {
use std::os::unix::ffi::OsStringExt; use std::os::unix::ffi::OsStringExt;
let (args, errs) = parse_reader(&b"\ let (args, errs) = parse_reader(
&b"\
quux quux
foo\xFFbar foo\xFFbar
baz baz
"[..]).unwrap(); "[..],
)
.unwrap();
assert!(errs.is_empty()); assert!(errs.is_empty());
assert_eq!(args, vec![ assert_eq!(
OsString::from("quux"), args,
OsString::from_vec(b"foo\xFFbar".to_vec()), vec![
OsString::from("baz"), OsString::from("quux"),
]); OsString::from_vec(b"foo\xFFbar".to_vec()),
OsString::from("baz"),
]
);
} }
// ... but test that invalid UTF-8 fails on Windows. // ... but test that invalid UTF-8 fails on Windows.
#[test] #[test]
#[cfg(not(unix))] #[cfg(not(unix))]
fn error() { fn error() {
let (args, errs) = parse_reader(&b"\ let (args, errs) = parse_reader(
&b"\
quux quux
foo\xFFbar foo\xFFbar
baz baz
"[..]).unwrap(); "[..],
)
.unwrap();
assert_eq!(errs.len(), 1); assert_eq!(errs.len(), 1);
assert_eq!(args, vec![ assert_eq!(args, vec![OsString::from("quux"), OsString::from("baz"),]);
OsString::from("quux"),
OsString::from("baz"),
]);
} }
} }

0
src/logger.rs → crates/core/logger.rs
View File

78
src/main.rs → crates/core/main.rs
View File

@@ -1,6 +1,7 @@
use std::error;
use std::io::{self, Write}; use std::io::{self, Write};
use std::process; use std::process;
use std::sync::{Arc, Mutex}; use std::sync::Mutex;
use std::time::Instant; use std::time::Instant;
use ignore::WalkState; use ignore::WalkState;
@@ -19,7 +20,30 @@ mod path_printer;
mod search; mod search;
mod subject; mod subject;
type Result<T> = ::std::result::Result<T, Box<::std::error::Error>>; // Since Rust no longer uses jemalloc by default, ripgrep will, by default,
// use the system allocator. On Linux, this would normally be glibc's
// allocator, which is pretty good. In particular, ripgrep does not have a
// particularly allocation heavy workload, so there really isn't much
// difference (for ripgrep's purposes) between glibc's allocator and jemalloc.
//
// However, when ripgrep is built with musl, this means ripgrep will use musl's
// allocator, which appears to be substantially worse. (musl's goal is not to
// have the fastest version of everything. Its goal is to be small and amenable
// to static compilation.) Even though ripgrep isn't particularly allocation
// heavy, musl's allocator appears to slow down ripgrep quite a bit. Therefore,
// when building with musl, we use jemalloc.
//
// We don't unconditionally use jemalloc because it can be nice to use the
// system's default allocator by default. Moreover, jemalloc seems to increase
// compilation times by a bit.
//
// Moreover, we only do this on 64-bit systems since jemalloc doesn't support
// i686.
#[cfg(all(target_env = "musl", target_pointer_width = "64"))]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
type Result<T> = ::std::result::Result<T, Box<dyn error::Error>>;
fn main() { fn main() {
if let Err(err) = Args::parse().and_then(try_main) { if let Err(err) = Args::parse().and_then(try_main) {
@@ -31,16 +55,15 @@ fn main() {
fn try_main(args: Args) -> Result<()> { fn try_main(args: Args) -> Result<()> {
use args::Command::*; use args::Command::*;
let matched = let matched = match args.command()? {
match args.command()? { Search => search(&args),
Search => search(&args), SearchParallel => search_parallel(&args),
SearchParallel => search_parallel(&args), SearchNever => Ok(false),
SearchNever => Ok(false), Files => files(&args),
Files => files(&args), FilesParallel => files_parallel(&args),
FilesParallel => files_parallel(&args), Types => types(&args),
Types => types(&args), PCRE2Version => pcre2_version(&args),
PCRE2Version => pcre2_version(&args), }?;
}?;
if matched && (args.quiet() || !messages::errored()) { if matched && (args.quiet() || !messages::errored()) {
process::exit(0) process::exit(0)
} else if messages::errored() { } else if messages::errored() {
@@ -102,24 +125,21 @@ fn search_parallel(args: &Args) -> Result<bool> {
let quit_after_match = args.quit_after_match()?; let quit_after_match = args.quit_after_match()?;
let started_at = Instant::now(); let started_at = Instant::now();
let subject_builder = Arc::new(args.subject_builder()); let subject_builder = args.subject_builder();
let bufwtr = Arc::new(args.buffer_writer()?); let bufwtr = args.buffer_writer()?;
let stats = Arc::new(args.stats()?.map(Mutex::new)); let stats = args.stats()?.map(Mutex::new);
let matched = Arc::new(AtomicBool::new(false)); let matched = AtomicBool::new(false);
let mut searcher_err = None; let mut searcher_err = None;
args.walker_parallel()?.run(|| { args.walker_parallel()?.run(|| {
let args = args.clone(); let bufwtr = &bufwtr;
let bufwtr = Arc::clone(&bufwtr); let stats = &stats;
let stats = Arc::clone(&stats); let matched = &matched;
let matched = Arc::clone(&matched); let subject_builder = &subject_builder;
let subject_builder = Arc::clone(&subject_builder);
let mut searcher = match args.search_worker(bufwtr.buffer()) { let mut searcher = match args.search_worker(bufwtr.buffer()) {
Ok(searcher) => searcher, Ok(searcher) => searcher,
Err(err) => { Err(err) => {
searcher_err = Some(err); searcher_err = Some(err);
return Box::new(move |_| { return Box::new(move |_| WalkState::Quit);
WalkState::Quit
});
} }
}; };
@@ -161,7 +181,7 @@ fn search_parallel(args: &Args) -> Result<bool> {
if let Some(err) = searcher_err.take() { if let Some(err) = searcher_err.take() {
return Err(err); return Err(err);
} }
if let Some(ref locked_stats) = *stats { if let Some(ref locked_stats) = stats {
let elapsed = Instant::now().duration_since(started_at); let elapsed = Instant::now().duration_since(started_at);
let stats = locked_stats.lock().unwrap(); let stats = locked_stats.lock().unwrap();
let mut searcher = args.search_worker(args.stdout())?; let mut searcher = args.search_worker(args.stdout())?;
@@ -211,9 +231,9 @@ fn files_parallel(args: &Args) -> Result<bool> {
use std::thread; use std::thread;
let quit_after_match = args.quit_after_match()?; let quit_after_match = args.quit_after_match()?;
let subject_builder = Arc::new(args.subject_builder()); let subject_builder = args.subject_builder();
let mut path_printer = args.path_printer(args.stdout())?; let mut path_printer = args.path_printer(args.stdout())?;
let matched = Arc::new(AtomicBool::new(false)); let matched = AtomicBool::new(false);
let (tx, rx) = mpsc::channel::<Subject>(); let (tx, rx) = mpsc::channel::<Subject>();
let print_thread = thread::spawn(move || -> io::Result<()> { let print_thread = thread::spawn(move || -> io::Result<()> {
@@ -223,8 +243,8 @@ fn files_parallel(args: &Args) -> Result<bool> {
Ok(()) Ok(())
}); });
args.walker_parallel()?.run(|| { args.walker_parallel()?.run(|| {
let subject_builder = Arc::clone(&subject_builder); let subject_builder = &subject_builder;
let matched = Arc::clone(&matched); let matched = &matched;
let tx = tx.clone(); let tx = tx.clone();
Box::new(move |result| { Box::new(move |result| {

0
src/messages.rs → crates/core/messages.rs
View File

5
src/path_printer.rs → crates/core/path_printer.rs
View File

@@ -37,10 +37,7 @@ impl PathPrinterBuilder {
/// Create a new path printer with the current configuration that writes /// Create a new path printer with the current configuration that writes
/// paths to the given writer. /// paths to the given writer.
pub fn build<W: WriteColor>(&self, wtr: W) -> PathPrinter<W> { pub fn build<W: WriteColor>(&self, wtr: W) -> PathPrinter<W> {
PathPrinter { PathPrinter { config: self.config.clone(), wtr }
config: self.config.clone(),
wtr: wtr,
}
} }
/// Set the color specification for this printer. /// Set the color specification for this printer.

106
src/search.rs → crates/core/search.rs
View File

@@ -7,9 +7,9 @@ use std::time::Duration;
use grep::cli; use grep::cli;
use grep::matcher::Matcher; use grep::matcher::Matcher;
#[cfg(feature = "pcre2")] #[cfg(feature = "pcre2")]
use grep::pcre2::{RegexMatcher as PCRE2RegexMatcher}; use grep::pcre2::RegexMatcher as PCRE2RegexMatcher;
use grep::printer::{JSON, Standard, Summary, Stats}; use grep::printer::{Standard, Stats, Summary, JSON};
use grep::regex::{RegexMatcher as RustRegexMatcher}; use grep::regex::RegexMatcher as RustRegexMatcher;
use grep::searcher::{BinaryDetection, Searcher}; use grep::searcher::{BinaryDetection, Searcher};
use ignore::overrides::Override; use ignore::overrides::Override;
use serde_json as json; use serde_json as json;
@@ -70,7 +70,7 @@ impl SearchWorkerBuilder {
SearchWorkerBuilder { SearchWorkerBuilder {
config: Config::default(), config: Config::default(),
command_builder: cmd_builder, command_builder: cmd_builder,
decomp_builder: decomp_builder, decomp_builder,
} }
} }
@@ -86,8 +86,12 @@ impl SearchWorkerBuilder {
let command_builder = self.command_builder.clone(); let command_builder = self.command_builder.clone();
let decomp_builder = self.decomp_builder.clone(); let decomp_builder = self.decomp_builder.clone();
SearchWorker { SearchWorker {
config, command_builder, decomp_builder, config,
matcher, searcher, printer, command_builder,
decomp_builder,
matcher,
searcher,
printer,
} }
} }
@@ -227,9 +231,7 @@ impl<W: WriteColor> Printer<W> {
stats: &Stats, stats: &Stats,
) -> io::Result<()> { ) -> io::Result<()> {
match *self { match *self {
Printer::JSON(_) => { Printer::JSON(_) => self.print_stats_json(total_duration, stats),
self.print_stats_json(total_duration, stats)
}
Printer::Standard(_) | Printer::Summary(_) => { Printer::Standard(_) | Printer::Summary(_) => {
self.print_stats_human(total_duration, stats) self.print_stats_human(total_duration, stats)
} }
@@ -273,17 +275,20 @@ impl<W: WriteColor> Printer<W> {
// the grep-printer crate. We simply "extend" it with the 'summary' // the grep-printer crate. We simply "extend" it with the 'summary'
// message type. // message type.
let fractional = fractional_seconds(total_duration); let fractional = fractional_seconds(total_duration);
json::to_writer(self.get_mut(), &json!({ json::to_writer(
"type": "summary", self.get_mut(),
"data": { &json!({
"stats": stats, "type": "summary",
"elapsed_total": { "data": {
"secs": total_duration.as_secs(), "stats": stats,
"nanos": total_duration.subsec_nanos(), "elapsed_total": {
"human": format!("{:0.6}s", fractional), "secs": total_duration.as_secs(),
}, "nanos": total_duration.subsec_nanos(),
} "human": format!("{:0.6}s", fractional),
}))?; },
}
}),
)?;
write!(self.get_mut(), "\n") write!(self.get_mut(), "\n")
} }
@@ -315,7 +320,23 @@ pub struct SearchWorker<W> {
impl<W: WriteColor> SearchWorker<W> { impl<W: WriteColor> SearchWorker<W> {
/// Execute a search over the given subject. /// Execute a search over the given subject.
pub fn search(&mut self, subject: &Subject) -> io::Result<SearchResult> { pub fn search(&mut self, subject: &Subject) -> io::Result<SearchResult> {
self.search_impl(subject) let bin = if subject.is_explicit() {
self.config.binary_explicit.clone()
} else {
self.config.binary_implicit.clone()
};
self.searcher.set_binary_detection(bin);
let path = subject.path();
if subject.is_stdin() {
self.search_reader(path, io::stdin().lock())
} else if self.should_preprocess(path) {
self.search_preprocessor(path)
} else if self.should_decompress(path) {
self.search_decompress(path)
} else {
self.search_path(path)
}
} }
/// Return a mutable reference to the underlying printer. /// Return a mutable reference to the underlying printer.
@@ -341,30 +362,6 @@ impl<W: WriteColor> SearchWorker<W> {
} }
} }
/// Search the given subject using the appropriate strategy.
fn search_impl(&mut self, subject: &Subject) -> io::Result<SearchResult> {
let bin =
if subject.is_explicit() {
self.config.binary_explicit.clone()
} else {
self.config.binary_implicit.clone()
};
self.searcher.set_binary_detection(bin);
let path = subject.path();
if subject.is_stdin() {
let stdin = io::stdin();
// A `return` here appeases the borrow checker. NLL will fix this.
return self.search_reader(path, stdin.lock());
} else if self.should_preprocess(path) {
self.search_preprocessor(path)
} else if self.should_decompress(path) {
self.search_decompress(path)
} else {
self.search_path(path)
}
}
/// Returns true if and only if the given file path should be /// Returns true if and only if the given file path should be
/// decompressed before searching. /// decompressed before searching.
fn should_decompress(&self, path: &Path) -> bool { fn should_decompress(&self, path: &Path) -> bool {
@@ -392,11 +389,19 @@ impl<W: WriteColor> SearchWorker<W> {
&mut self, &mut self,
path: &Path, path: &Path,
) -> io::Result<SearchResult> { ) -> io::Result<SearchResult> {
let bin = self.config.preprocessor.clone().unwrap(); let bin = self.config.preprocessor.as_ref().unwrap();
let mut cmd = Command::new(&bin); let mut cmd = Command::new(bin);
cmd.arg(path).stdin(Stdio::from(File::open(path)?)); cmd.arg(path).stdin(Stdio::from(File::open(path)?));
let rdr = self.command_builder.build(&mut cmd)?; let rdr = self.command_builder.build(&mut cmd).map_err(|err| {
io::Error::new(
io::ErrorKind::Other,
format!(
"preprocessor command could not start: '{:?}': {}",
cmd, err,
),
)
})?;
self.search_reader(path, rdr).map_err(|err| { self.search_reader(path, rdr).map_err(|err| {
io::Error::new( io::Error::new(
io::ErrorKind::Other, io::ErrorKind::Other,
@@ -408,10 +413,7 @@ impl<W: WriteColor> SearchWorker<W> {
/// Attempt to decompress the data at the given file path and search the /// Attempt to decompress the data at the given file path and search the
/// result. If the given file path isn't recognized as a compressed file, /// result. If the given file path isn't recognized as a compressed file,
/// then search it without doing any decompression. /// then search it without doing any decompression.
fn search_decompress( fn search_decompress(&mut self, path: &Path) -> io::Result<SearchResult> {
&mut self,
path: &Path,
) -> io::Result<SearchResult> {
let rdr = self.decomp_builder.build(path)?; let rdr = self.decomp_builder.build(path)?;
self.search_reader(path, rdr) self.search_reader(path, rdr)
} }

16
src/subject.rs → crates/core/subject.rs
View File

@@ -11,9 +11,7 @@ struct Config {
impl Default for Config { impl Default for Config {
fn default() -> Config { fn default() -> Config {
Config { Config { strip_dot_prefix: false }
strip_dot_prefix: false,
}
} }
} }
@@ -52,10 +50,8 @@ impl SubjectBuilder {
/// If a subject could not be created or should otherwise not be searched, /// If a subject could not be created or should otherwise not be searched,
/// then this returns `None` after emitting any relevant log messages. /// then this returns `None` after emitting any relevant log messages.
pub fn build(&self, dent: DirEntry) -> Option<Subject> { pub fn build(&self, dent: DirEntry) -> Option<Subject> {
let subj = Subject { let subj =
dent: dent, Subject { dent, strip_dot_prefix: self.config.strip_dot_prefix };
strip_dot_prefix: self.config.strip_dot_prefix,
};
if let Some(ignore_err) = subj.dent.error() { if let Some(ignore_err) = subj.dent.error() {
ignore_message!("{}", ignore_err); ignore_message!("{}", ignore_err);
} }
@@ -78,9 +74,9 @@ impl SubjectBuilder {
log::debug!( log::debug!(
"ignoring {}: failed to pass subject filter: \ "ignoring {}: failed to pass subject filter: \
file type: {:?}, metadata: {:?}", file type: {:?}, metadata: {:?}",
subj.dent.path().display(), subj.dent.path().display(),
subj.dent.file_type(), subj.dent.file_type(),
subj.dent.metadata() subj.dent.metadata()
); );
} }
None None

0
globset/COPYING → crates/globset/COPYING
View File

12
globset/Cargo.toml → crates/globset/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package] [package]
name = "globset" name = "globset"
version = "0.4.3" #:version version = "0.4.5" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"] authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """ description = """
Cross platform single glob and glob set matching. Glob set matching is the Cross platform single glob and glob set matching. Glob set matching is the
@@ -8,8 +8,8 @@ process of matching one or more glob patterns against a single candidate path
simultaneously, and returning all of the globs that matched. simultaneously, and returning all of the globs that matched.
""" """
documentation = "https://docs.rs/globset" documentation = "https://docs.rs/globset"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/globset" homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/globset"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/globset" repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/globset"
readme = "README.md" readme = "README.md"
keywords = ["regex", "glob", "multiple", "set", "pattern"] keywords = ["regex", "glob", "multiple", "set", "pattern"]
license = "Unlicense/MIT" license = "Unlicense/MIT"
@@ -20,13 +20,17 @@ bench = false
[dependencies] [dependencies]
aho-corasick = "0.7.3" aho-corasick = "0.7.3"
bstr = { version = "0.1.2", default-features = false, features = ["std"] } bstr = { version = "0.2.0", default-features = false, features = ["std"] }
fnv = "1.0.6" fnv = "1.0.6"
log = "0.4.5" log = "0.4.5"
regex = "1.1.5" regex = "1.1.5"
serde = { version = "1.0.104", optional = true }
[dev-dependencies] [dev-dependencies]
glob = "0.3.0" glob = "0.3.0"
lazy_static = "1"
serde_json = "1.0.45"
[features] [features]
simd-accel = [] simd-accel = []
serde1 = ["serde"]

0
grep-cli/LICENSE-MIT → crates/globset/LICENSE-MIT
View File

4
globset/README.md → crates/globset/README.md
View File

@@ -29,6 +29,10 @@ and this to your crate root:
extern crate globset; extern crate globset;
``` ```
### Features
* `serde1`: Enables implementing Serde traits on the `Glob` type.
### Example: one glob ### Example: one glob
This example shows how to match a single glob against a single file path. This example shows how to match a single glob against a single file path.

0
grep-cli/UNLICENSE → crates/globset/UNLICENSE
View File

5
globset/benches/bench.rs → crates/globset/benches/bench.rs
View File

@@ -6,14 +6,9 @@ tool itself, see the benchsuite directory.
extern crate glob; extern crate glob;
extern crate globset; extern crate globset;
#[macro_use]
extern crate lazy_static;
extern crate regex; extern crate regex;
extern crate test; extern crate test;
use std::ffi::OsStr;
use std::path::Path;
use globset::{Candidate, Glob, GlobMatcher, GlobSet, GlobSetBuilder}; use globset::{Candidate, Glob, GlobMatcher, GlobSet, GlobSetBuilder};
const EXT: &'static str = "some/a/bigger/path/to/the/crazy/needle.txt"; const EXT: &'static str = "some/a/bigger/path/to/the/crazy/needle.txt";

248
globset/src/glob.rs → crates/globset/src/glob.rs
View File

@@ -2,13 +2,13 @@ use std::fmt;
use std::hash; use std::hash;
use std::iter; use std::iter;
use std::ops::{Deref, DerefMut}; use std::ops::{Deref, DerefMut};
use std::path::{Path, is_separator}; use std::path::{is_separator, Path};
use std::str; use std::str;
use regex; use regex;
use regex::bytes::Regex; use regex::bytes::Regex;
use {Candidate, Error, ErrorKind, new_regex}; use {new_regex, Candidate, Error, ErrorKind};
/// Describes a matching strategy for a particular pattern. /// Describes a matching strategy for a particular pattern.
/// ///
@@ -85,16 +85,16 @@ pub struct Glob {
} }
impl PartialEq for Glob { impl PartialEq for Glob {
fn eq(&self, other: &Glob) -> bool { fn eq(&self, other: &Glob) -> bool {
self.glob == other.glob && self.opts == other.opts self.glob == other.glob && self.opts == other.opts
} }
} }
impl hash::Hash for Glob { impl hash::Hash for Glob {
fn hash<H: hash::Hasher>(&self, state: &mut H) { fn hash<H: hash::Hasher>(&self, state: &mut H) {
self.glob.hash(state); self.glob.hash(state);
self.opts.hash(state); self.opts.hash(state);
} }
} }
impl fmt::Display for Glob { impl fmt::Display for Glob {
@@ -103,6 +103,14 @@ impl fmt::Display for Glob {
} }
} }
impl str::FromStr for Glob {
type Err = Error;
fn from_str(glob: &str) -> Result<Self, Self::Err> {
Self::new(glob)
}
}
/// A matcher for a single pattern. /// A matcher for a single pattern.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct GlobMatcher { pub struct GlobMatcher {
@@ -120,7 +128,12 @@ impl GlobMatcher {
/// Tests whether the given path matches this pattern or not. /// Tests whether the given path matches this pattern or not.
pub fn is_match_candidate(&self, path: &Candidate) -> bool { pub fn is_match_candidate(&self, path: &Candidate) -> bool {
self.re.is_match(path.path.as_bytes()) self.re.is_match(&path.path)
}
/// Returns the `Glob` used to compile this matcher.
pub fn glob(&self) -> &Glob {
&self.pat
} }
} }
@@ -145,7 +158,7 @@ impl GlobStrategic {
/// Tests whether the given path matches this pattern or not. /// Tests whether the given path matches this pattern or not.
fn is_match_candidate(&self, candidate: &Candidate) -> bool { fn is_match_candidate(&self, candidate: &Candidate) -> bool {
let byte_path = candidate.path.as_bytes(); let byte_path = &*candidate.path;
match self.strategy { match self.strategy {
MatchStrategy::Literal(ref lit) => lit.as_bytes() == byte_path, MatchStrategy::Literal(ref lit) => lit.as_bytes() == byte_path,
@@ -214,11 +227,15 @@ struct Tokens(Vec<Token>);
impl Deref for Tokens { impl Deref for Tokens {
type Target = Vec<Token>; type Target = Vec<Token>;
fn deref(&self) -> &Vec<Token> { &self.0 } fn deref(&self) -> &Vec<Token> {
&self.0
}
} }
impl DerefMut for Tokens { impl DerefMut for Tokens {
fn deref_mut(&mut self) -> &mut Vec<Token> { &mut self.0 } fn deref_mut(&mut self) -> &mut Vec<Token> {
&mut self.0
}
} }
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
@@ -229,10 +246,7 @@ enum Token {
RecursivePrefix, RecursivePrefix,
RecursiveSuffix, RecursiveSuffix,
RecursiveZeroOrMore, RecursiveZeroOrMore,
Class { Class { negated: bool, ranges: Vec<(char, char)> },
negated: bool,
ranges: Vec<(char, char)>,
},
Alternates(Vec<Tokens>), Alternates(Vec<Tokens>),
} }
@@ -244,12 +258,9 @@ impl Glob {
/// Returns a matcher for this pattern. /// Returns a matcher for this pattern.
pub fn compile_matcher(&self) -> GlobMatcher { pub fn compile_matcher(&self) -> GlobMatcher {
let re = new_regex(&self.re) let re =
.expect("regex compilation shouldn't fail"); new_regex(&self.re).expect("regex compilation shouldn't fail");
GlobMatcher { GlobMatcher { pat: self.clone(), re: re }
pat: self.clone(),
re: re,
}
} }
/// Returns a strategic matcher. /// Returns a strategic matcher.
@@ -260,13 +271,9 @@ impl Glob {
#[cfg(test)] #[cfg(test)]
fn compile_strategic_matcher(&self) -> GlobStrategic { fn compile_strategic_matcher(&self) -> GlobStrategic {
let strategy = MatchStrategy::new(self); let strategy = MatchStrategy::new(self);
let re = new_regex(&self.re) let re =
.expect("regex compilation shouldn't fail"); new_regex(&self.re).expect("regex compilation shouldn't fail");
GlobStrategic { GlobStrategic { strategy: strategy, pat: self.clone(), re: re }
strategy: strategy,
pat: self.clone(),
re: re,
}
} }
/// Returns the original glob pattern used to build this pattern. /// Returns the original glob pattern used to build this pattern.
@@ -524,7 +531,7 @@ impl Glob {
| Token::RecursiveZeroOrMore => { | Token::RecursiveZeroOrMore => {
return None; return None;
} }
Token::Class{..} | Token::Alternates(..) => { Token::Class { .. } | Token::Alternates(..) => {
// We *could* be a little smarter here, but either one // We *could* be a little smarter here, but either one
// of these is going to prevent our literal optimizations // of these is going to prevent our literal optimizations
// anyway, so give up. // anyway, so give up.
@@ -561,10 +568,7 @@ impl<'a> GlobBuilder<'a> {
/// ///
/// The pattern is not compiled until `build` is called. /// The pattern is not compiled until `build` is called.
pub fn new(glob: &'a str) -> GlobBuilder<'a> { pub fn new(glob: &'a str) -> GlobBuilder<'a> {
GlobBuilder { GlobBuilder { glob: glob, opts: GlobOptions::default() }
glob: glob,
opts: GlobOptions::default(),
}
} }
/// Parses and builds the pattern. /// Parses and builds the pattern.
@@ -862,25 +866,22 @@ impl<'a> Parser<'a> {
return Ok(()); return Ok(());
} }
} }
let is_suffix = let is_suffix = match self.peek() {
match self.peek() { None => {
None => { assert!(self.bump().is_none());
assert!(self.bump().is_none()); true
true }
} Some(',') | Some('}') if self.stack.len() >= 2 => true,
Some(',') | Some('}') if self.stack.len() >= 2 => { Some(c) if is_separator(c) => {
true assert!(self.bump().map(is_separator).unwrap_or(false));
} false
Some(c) if is_separator(c) => { }
assert!(self.bump().map(is_separator).unwrap_or(false)); _ => {
false self.push_token(Token::ZeroOrMore)?;
} self.push_token(Token::ZeroOrMore)?;
_ => { return Ok(());
self.push_token(Token::ZeroOrMore)?; }
self.push_token(Token::ZeroOrMore)?; };
return Ok(());
}
};
match self.pop_token()? { match self.pop_token()? {
Token::RecursivePrefix => { Token::RecursivePrefix => {
self.push_token(Token::RecursivePrefix)?; self.push_token(Token::RecursivePrefix)?;
@@ -960,7 +961,10 @@ impl<'a> Parser<'a> {
// invariant: in_range is only set when there is // invariant: in_range is only set when there is
// already at least one character seen. // already at least one character seen.
add_to_last_range( add_to_last_range(
&self.glob, ranges.last_mut().unwrap(), c)?; &self.glob,
ranges.last_mut().unwrap(),
c,
)?;
} else { } else {
ranges.push((c, c)); ranges.push((c, c));
} }
@@ -974,10 +978,7 @@ impl<'a> Parser<'a> {
// it as a literal. // it as a literal.
ranges.push(('-', '-')); ranges.push(('-', '-'));
} }
self.push_token(Token::Class { self.push_token(Token::Class { negated: negated, ranges: ranges })
negated: negated,
ranges: ranges,
})
} }
fn bump(&mut self) -> Option<char> { fn bump(&mut self) -> Option<char> {
@@ -1006,9 +1007,9 @@ fn ends_with(needle: &[u8], haystack: &[u8]) -> bool {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use {GlobSetBuilder, ErrorKind};
use super::{Glob, GlobBuilder, Token};
use super::Token::*; use super::Token::*;
use super::{Glob, GlobBuilder, Token};
use {ErrorKind, GlobSetBuilder};
#[derive(Clone, Copy, Debug, Default)] #[derive(Clone, Copy, Debug, Default)]
struct Options { struct Options {
@@ -1024,7 +1025,7 @@ mod tests {
let pat = Glob::new($pat).unwrap(); let pat = Glob::new($pat).unwrap();
assert_eq!($tokens, pat.tokens.0); assert_eq!($tokens, pat.tokens.0);
} }
} };
} }
macro_rules! syntaxerr { macro_rules! syntaxerr {
@@ -1034,7 +1035,7 @@ mod tests {
let err = Glob::new($pat).unwrap_err(); let err = Glob::new($pat).unwrap_err();
assert_eq!(&$err, err.kind()); assert_eq!(&$err, err.kind());
} }
} };
} }
macro_rules! toregex { macro_rules! toregex {
@@ -1116,7 +1117,9 @@ mod tests {
}; };
} }
fn s(string: &str) -> String { string.to_string() } fn s(string: &str) -> String {
string.to_string()
}
fn class(s: char, e: char) -> Token { fn class(s: char, e: char) -> Token {
Class { negated: false, ranges: vec![(s, e)] } Class { negated: false, ranges: vec![(s, e)] }
@@ -1140,16 +1143,20 @@ mod tests {
syntax!(any2, "a?b", vec![Literal('a'), Any, Literal('b')]); syntax!(any2, "a?b", vec![Literal('a'), Any, Literal('b')]);
syntax!(seq1, "*", vec![ZeroOrMore]); syntax!(seq1, "*", vec![ZeroOrMore]);
syntax!(seq2, "a*b", vec![Literal('a'), ZeroOrMore, Literal('b')]); syntax!(seq2, "a*b", vec![Literal('a'), ZeroOrMore, Literal('b')]);
syntax!(seq3, "*a*b*", vec![ syntax!(
ZeroOrMore, Literal('a'), ZeroOrMore, Literal('b'), ZeroOrMore, seq3,
]); "*a*b*",
vec![ZeroOrMore, Literal('a'), ZeroOrMore, Literal('b'), ZeroOrMore,]
);
syntax!(rseq1, "**", vec![RecursivePrefix]); syntax!(rseq1, "**", vec![RecursivePrefix]);
syntax!(rseq2, "**/", vec![RecursivePrefix]); syntax!(rseq2, "**/", vec![RecursivePrefix]);
syntax!(rseq3, "/**", vec![RecursiveSuffix]); syntax!(rseq3, "/**", vec![RecursiveSuffix]);
syntax!(rseq4, "/**/", vec![RecursiveZeroOrMore]); syntax!(rseq4, "/**/", vec![RecursiveZeroOrMore]);
syntax!(rseq5, "a/**/b", vec![ syntax!(
Literal('a'), RecursiveZeroOrMore, Literal('b'), rseq5,
]); "a/**/b",
vec![Literal('a'), RecursiveZeroOrMore, Literal('b'),]
);
syntax!(cls1, "[a]", vec![class('a', 'a')]); syntax!(cls1, "[a]", vec![class('a', 'a')]);
syntax!(cls2, "[!a]", vec![classn('a', 'a')]); syntax!(cls2, "[!a]", vec![classn('a', 'a')]);
syntax!(cls3, "[a-z]", vec![class('a', 'z')]); syntax!(cls3, "[a-z]", vec![class('a', 'z')]);
@@ -1161,9 +1168,11 @@ mod tests {
syntax!(cls9, "[a-]", vec![rclass(&[('a', 'a'), ('-', '-')])]); syntax!(cls9, "[a-]", vec![rclass(&[('a', 'a'), ('-', '-')])]);
syntax!(cls10, "[-a-z]", vec![rclass(&[('-', '-'), ('a', 'z')])]); syntax!(cls10, "[-a-z]", vec![rclass(&[('-', '-'), ('a', 'z')])]);
syntax!(cls11, "[a-z-]", vec![rclass(&[('a', 'z'), ('-', '-')])]); syntax!(cls11, "[a-z-]", vec![rclass(&[('a', 'z'), ('-', '-')])]);
syntax!(cls12, "[-a-z-]", vec![ syntax!(
rclass(&[('-', '-'), ('a', 'z'), ('-', '-')]), cls12,
]); "[-a-z-]",
vec![rclass(&[('-', '-'), ('a', 'z'), ('-', '-')]),]
);
syntax!(cls13, "[]-z]", vec![class(']', 'z')]); syntax!(cls13, "[]-z]", vec![class(']', 'z')]);
syntax!(cls14, "[--z]", vec![class('-', 'z')]); syntax!(cls14, "[--z]", vec![class('-', 'z')]);
syntax!(cls15, "[ --]", vec![class(' ', '-')]); syntax!(cls15, "[ --]", vec![class(' ', '-')]);
@@ -1181,26 +1190,14 @@ mod tests {
syntaxerr!(err_range1, "[z-a]", ErrorKind::InvalidRange('z', 'a')); syntaxerr!(err_range1, "[z-a]", ErrorKind::InvalidRange('z', 'a'));
syntaxerr!(err_range2, "[z--]", ErrorKind::InvalidRange('z', '-')); syntaxerr!(err_range2, "[z--]", ErrorKind::InvalidRange('z', '-'));
const CASEI: Options = Options { const CASEI: Options =
casei: Some(true), Options { casei: Some(true), litsep: None, bsesc: None };
litsep: None, const SLASHLIT: Options =
bsesc: None, Options { casei: None, litsep: Some(true), bsesc: None };
}; const NOBSESC: Options =
const SLASHLIT: Options = Options { Options { casei: None, litsep: None, bsesc: Some(false) };
casei: None, const BSESC: Options =
litsep: Some(true), Options { casei: None, litsep: None, bsesc: Some(true) };
bsesc: None,
};
const NOBSESC: Options = Options {
casei: None,
litsep: None,
bsesc: Some(false),
};
const BSESC: Options = Options {
casei: None,
litsep: None,
bsesc: Some(true),
};
toregex!(re_casei, "a", "(?i)^a$", &CASEI); toregex!(re_casei, "a", "(?i)^a$", &CASEI);
@@ -1298,8 +1295,11 @@ mod tests {
matches!(matchpat4, "*hello.txt", "some\\path\\to\\hello.txt"); matches!(matchpat4, "*hello.txt", "some\\path\\to\\hello.txt");
matches!(matchpat5, "*hello.txt", "/an/absolute/path/to/hello.txt"); matches!(matchpat5, "*hello.txt", "/an/absolute/path/to/hello.txt");
matches!(matchpat6, "*some/path/to/hello.txt", "some/path/to/hello.txt"); matches!(matchpat6, "*some/path/to/hello.txt", "some/path/to/hello.txt");
matches!(matchpat7, "*some/path/to/hello.txt", matches!(
"a/bigger/some/path/to/hello.txt"); matchpat7,
"*some/path/to/hello.txt",
"a/bigger/some/path/to/hello.txt"
);
matches!(matchescape, "_[[]_[]]_[?]_[*]_!_", "_[_]_?_*_!_"); matches!(matchescape, "_[[]_[]]_[?]_[*]_!_", "_[_]_?_*_!_");
@@ -1362,28 +1362,44 @@ mod tests {
nmatches!(matchnot15, "[!-]", "-"); nmatches!(matchnot15, "[!-]", "-");
nmatches!(matchnot16, "*hello.txt", "hello.txt-and-then-some"); nmatches!(matchnot16, "*hello.txt", "hello.txt-and-then-some");
nmatches!(matchnot17, "*hello.txt", "goodbye.txt"); nmatches!(matchnot17, "*hello.txt", "goodbye.txt");
nmatches!(matchnot18, "*some/path/to/hello.txt", nmatches!(
"some/path/to/hello.txt-and-then-some"); matchnot18,
nmatches!(matchnot19, "*some/path/to/hello.txt", "*some/path/to/hello.txt",
"some/other/path/to/hello.txt"); "some/path/to/hello.txt-and-then-some"
);
nmatches!(
matchnot19,
"*some/path/to/hello.txt",
"some/other/path/to/hello.txt"
);
nmatches!(matchnot20, "a", "foo/a"); nmatches!(matchnot20, "a", "foo/a");
nmatches!(matchnot21, "./foo", "foo"); nmatches!(matchnot21, "./foo", "foo");
nmatches!(matchnot22, "**/foo", "foofoo"); nmatches!(matchnot22, "**/foo", "foofoo");
nmatches!(matchnot23, "**/foo/bar", "foofoo/bar"); nmatches!(matchnot23, "**/foo/bar", "foofoo/bar");
nmatches!(matchnot24, "/*.c", "mozilla-sha1/sha1.c"); nmatches!(matchnot24, "/*.c", "mozilla-sha1/sha1.c");
nmatches!(matchnot25, "*.c", "mozilla-sha1/sha1.c", SLASHLIT); nmatches!(matchnot25, "*.c", "mozilla-sha1/sha1.c", SLASHLIT);
nmatches!(matchnot26, "**/m4/ltoptions.m4", nmatches!(
"csharp/src/packages/repositories.config", SLASHLIT); matchnot26,
"**/m4/ltoptions.m4",
"csharp/src/packages/repositories.config",
SLASHLIT
);
nmatches!(matchnot27, "a[^0-9]b", "a0b"); nmatches!(matchnot27, "a[^0-9]b", "a0b");
nmatches!(matchnot28, "a[^0-9]b", "a9b"); nmatches!(matchnot28, "a[^0-9]b", "a9b");
nmatches!(matchnot29, "[^-]", "-"); nmatches!(matchnot29, "[^-]", "-");
nmatches!(matchnot30, "some/*/needle.txt", "some/needle.txt"); nmatches!(matchnot30, "some/*/needle.txt", "some/needle.txt");
nmatches!( nmatches!(
matchrec31, matchrec31,
"some/*/needle.txt", "some/one/two/needle.txt", SLASHLIT); "some/*/needle.txt",
"some/one/two/needle.txt",
SLASHLIT
);
nmatches!( nmatches!(
matchrec32, matchrec32,
"some/*/needle.txt", "some/one/two/three/needle.txt", SLASHLIT); "some/*/needle.txt",
"some/one/two/three/needle.txt",
SLASHLIT
);
macro_rules! extract { macro_rules! extract {
($which:ident, $name:ident, $pat:expr, $expect:expr) => { ($which:ident, $name:ident, $pat:expr, $expect:expr) => {
@@ -1445,19 +1461,27 @@ mod tests {
literal!(extract_lit7, "foo/bar", Some(s("foo/bar"))); literal!(extract_lit7, "foo/bar", Some(s("foo/bar")));
literal!(extract_lit8, "**/foo/bar", None); literal!(extract_lit8, "**/foo/bar", None);
basetokens!(extract_basetoks1, "**/foo", Some(&*vec![ basetokens!(
Literal('f'), Literal('o'), Literal('o'), extract_basetoks1,
])); "**/foo",
Some(&*vec![Literal('f'), Literal('o'), Literal('o'),])
);
basetokens!(extract_basetoks2, "**/foo", None, CASEI); basetokens!(extract_basetoks2, "**/foo", None, CASEI);
basetokens!(extract_basetoks3, "**/foo", Some(&*vec![ basetokens!(
Literal('f'), Literal('o'), Literal('o'), extract_basetoks3,
]), SLASHLIT); "**/foo",
Some(&*vec![Literal('f'), Literal('o'), Literal('o'),]),
SLASHLIT
);
basetokens!(extract_basetoks4, "*foo", None, SLASHLIT); basetokens!(extract_basetoks4, "*foo", None, SLASHLIT);
basetokens!(extract_basetoks5, "*foo", None); basetokens!(extract_basetoks5, "*foo", None);
basetokens!(extract_basetoks6, "**/fo*o", None); basetokens!(extract_basetoks6, "**/fo*o", None);
basetokens!(extract_basetoks7, "**/fo*o", Some(&*vec![ basetokens!(
Literal('f'), Literal('o'), ZeroOrMore, Literal('o'), extract_basetoks7,
]), SLASHLIT); "**/fo*o",
Some(&*vec![Literal('f'), Literal('o'), ZeroOrMore, Literal('o'),]),
SLASHLIT
);
ext!(extract_ext1, "**/*.rs", Some(s(".rs"))); ext!(extract_ext1, "**/*.rs", Some(s(".rs")));
ext!(extract_ext2, "**/*.rs.bak", None); ext!(extract_ext2, "**/*.rs.bak", None);

89
globset/src/lib.rs → crates/globset/src/lib.rs
View File

@@ -110,6 +110,9 @@ extern crate fnv;
extern crate log; extern crate log;
extern crate regex; extern crate regex;
#[cfg(feature = "serde1")]
extern crate serde;
use std::borrow::Cow; use std::borrow::Cow;
use std::collections::{BTreeMap, HashMap}; use std::collections::{BTreeMap, HashMap};
use std::error::Error as StdError; use std::error::Error as StdError;
@@ -119,16 +122,19 @@ use std::path::Path;
use std::str; use std::str;
use aho_corasick::AhoCorasick; use aho_corasick::AhoCorasick;
use bstr::{B, BStr, BString}; use bstr::{ByteSlice, ByteVec, B};
use regex::bytes::{Regex, RegexBuilder, RegexSet}; use regex::bytes::{Regex, RegexBuilder, RegexSet};
use pathutil::{file_name, file_name_ext, normalize_path};
use glob::MatchStrategy; use glob::MatchStrategy;
pub use glob::{Glob, GlobBuilder, GlobMatcher}; pub use glob::{Glob, GlobBuilder, GlobMatcher};
use pathutil::{file_name, file_name_ext, normalize_path};
mod glob; mod glob;
mod pathutil; mod pathutil;
#[cfg(feature = "serde1")]
mod serde_impl;
/// Represents an error that can occur when parsing a glob pattern. /// Represents an error that can occur when parsing a glob pattern.
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub struct Error { pub struct Error {
@@ -202,9 +208,7 @@ impl ErrorKind {
ErrorKind::UnclosedClass => { ErrorKind::UnclosedClass => {
"unclosed character class; missing ']'" "unclosed character class; missing ']'"
} }
ErrorKind::InvalidRange(_, _) => { ErrorKind::InvalidRange(_, _) => "invalid character range",
"invalid character range"
}
ErrorKind::UnopenedAlternates => { ErrorKind::UnopenedAlternates => {
"unopened alternate group; missing '{' \ "unopened alternate group; missing '{' \
(maybe escape '}' with '[}]'?)" (maybe escape '}' with '[}]'?)"
@@ -216,9 +220,7 @@ impl ErrorKind {
ErrorKind::NestedAlternates => { ErrorKind::NestedAlternates => {
"nested alternate groups are not allowed" "nested alternate groups are not allowed"
} }
ErrorKind::DanglingEscape => { ErrorKind::DanglingEscape => "dangling '\\'",
"dangling '\\'"
}
ErrorKind::Regex(ref err) => err, ErrorKind::Regex(ref err) => err,
ErrorKind::__Nonexhaustive => unreachable!(), ErrorKind::__Nonexhaustive => unreachable!(),
} }
@@ -245,9 +247,7 @@ impl fmt::Display for ErrorKind {
| ErrorKind::UnclosedAlternates | ErrorKind::UnclosedAlternates
| ErrorKind::NestedAlternates | ErrorKind::NestedAlternates
| ErrorKind::DanglingEscape | ErrorKind::DanglingEscape
| ErrorKind::Regex(_) => { | ErrorKind::Regex(_) => write!(f, "{}", self.description()),
write!(f, "{}", self.description())
}
ErrorKind::InvalidRange(s, e) => { ErrorKind::InvalidRange(s, e) => {
write!(f, "invalid range; '{}' > '{}'", s, e) write!(f, "invalid range; '{}' > '{}'", s, e)
} }
@@ -262,21 +262,20 @@ fn new_regex(pat: &str) -> Result<Regex, Error> {
.size_limit(10 * (1 << 20)) .size_limit(10 * (1 << 20))
.dfa_size_limit(10 * (1 << 20)) .dfa_size_limit(10 * (1 << 20))
.build() .build()
.map_err(|err| { .map_err(|err| Error {
Error { glob: Some(pat.to_string()),
glob: Some(pat.to_string()), kind: ErrorKind::Regex(err.to_string()),
kind: ErrorKind::Regex(err.to_string()),
}
}) })
} }
fn new_regex_set<I, S>(pats: I) -> Result<RegexSet, Error> fn new_regex_set<I, S>(pats: I) -> Result<RegexSet, Error>
where S: AsRef<str>, I: IntoIterator<Item=S> { where
RegexSet::new(pats).map_err(|err| { S: AsRef<str>,
Error { I: IntoIterator<Item = S>,
glob: None, {
kind: ErrorKind::Regex(err.to_string()), RegexSet::new(pats).map_err(|err| Error {
} glob: None,
kind: ErrorKind::Regex(err.to_string()),
}) })
} }
@@ -294,10 +293,7 @@ impl GlobSet {
/// Create an empty `GlobSet`. An empty set matches nothing. /// Create an empty `GlobSet`. An empty set matches nothing.
#[inline] #[inline]
pub fn empty() -> GlobSet { pub fn empty() -> GlobSet {
GlobSet { GlobSet { len: 0, strats: vec![] }
len: 0,
strats: vec![],
}
} }
/// Returns true if this set is empty, and therefore matches nothing. /// Returns true if this set is empty, and therefore matches nothing.
@@ -432,11 +428,17 @@ impl GlobSet {
} }
} }
} }
debug!("built glob set; {} literals, {} basenames, {} extensions, \ debug!(
"built glob set; {} literals, {} basenames, {} extensions, \
{} prefixes, {} suffixes, {} required extensions, {} regexes", {} prefixes, {} suffixes, {} required extensions, {} regexes",
lits.0.len(), base_lits.0.len(), exts.0.len(), lits.0.len(),
prefixes.literals.len(), suffixes.literals.len(), base_lits.0.len(),
required_exts.0.len(), regexes.literals.len()); exts.0.len(),
prefixes.literals.len(),
suffixes.literals.len(),
required_exts.0.len(),
regexes.literals.len()
);
Ok(GlobSet { Ok(GlobSet {
len: pats.len(), len: pats.len(),
strats: vec![ strats: vec![
@@ -446,7 +448,8 @@ impl GlobSet {
GlobSetMatchStrategy::Suffix(suffixes.suffix()), GlobSetMatchStrategy::Suffix(suffixes.suffix()),
GlobSetMatchStrategy::Prefix(prefixes.prefix()), GlobSetMatchStrategy::Prefix(prefixes.prefix()),
GlobSetMatchStrategy::RequiredExtension( GlobSetMatchStrategy::RequiredExtension(
required_exts.build()?), required_exts.build()?,
),
GlobSetMatchStrategy::Regex(regexes.regex_set()?), GlobSetMatchStrategy::Regex(regexes.regex_set()?),
], ],
}) })
@@ -490,25 +493,21 @@ impl GlobSetBuilder {
/// path against multiple globs or sets of globs. /// path against multiple globs or sets of globs.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct Candidate<'a> { pub struct Candidate<'a> {
path: Cow<'a, BStr>, path: Cow<'a, [u8]>,
basename: Cow<'a, BStr>, basename: Cow<'a, [u8]>,
ext: Cow<'a, BStr>, ext: Cow<'a, [u8]>,
} }
impl<'a> Candidate<'a> { impl<'a> Candidate<'a> {
/// Create a new candidate for matching from the given path. /// Create a new candidate for matching from the given path.
pub fn new<P: AsRef<Path> + ?Sized>(path: &'a P) -> Candidate<'a> { pub fn new<P: AsRef<Path> + ?Sized>(path: &'a P) -> Candidate<'a> {
let path = normalize_path(BString::from_path_lossy(path.as_ref())); let path = normalize_path(Vec::from_path_lossy(path.as_ref()));
let basename = file_name(&path).unwrap_or(Cow::Borrowed(B(""))); let basename = file_name(&path).unwrap_or(Cow::Borrowed(B("")));
let ext = file_name_ext(&basename).unwrap_or(Cow::Borrowed(B(""))); let ext = file_name_ext(&basename).unwrap_or(Cow::Borrowed(B("")));
Candidate { Candidate { path: path, basename: basename, ext: ext }
path: path,
basename: basename,
ext: ext,
}
} }
fn path_prefix(&self, max: usize) -> &BStr { fn path_prefix(&self, max: usize) -> &[u8] {
if self.path.len() <= max { if self.path.len() <= max {
&*self.path &*self.path
} else { } else {
@@ -516,7 +515,7 @@ impl<'a> Candidate<'a> {
} }
} }
fn path_suffix(&self, max: usize) -> &BStr { fn path_suffix(&self, max: usize) -> &[u8] {
if self.path.len() <= max { if self.path.len() <= max {
&*self.path &*self.path
} else { } else {
@@ -767,11 +766,7 @@ struct MultiStrategyBuilder {
impl MultiStrategyBuilder { impl MultiStrategyBuilder {
fn new() -> MultiStrategyBuilder { fn new() -> MultiStrategyBuilder {
MultiStrategyBuilder { MultiStrategyBuilder { literals: vec![], map: vec![], longest: 0 }
literals: vec![],
map: vec![],
longest: 0,
}
} }
fn add(&mut self, global_index: usize, literal: String) { fn add(&mut self, global_index: usize, literal: String) {

18
globset/src/pathutil.rs → crates/globset/src/pathutil.rs
View File

@@ -1,15 +1,15 @@
use std::borrow::Cow; use std::borrow::Cow;
use bstr::BStr; use bstr::{ByteSlice, ByteVec};
/// The final component of the path, if it is a normal file. /// The final component of the path, if it is a normal file.
/// ///
/// If the path terminates in ., .., or consists solely of a root of prefix, /// If the path terminates in ., .., or consists solely of a root of prefix,
/// file_name will return None. /// file_name will return None.
pub fn file_name<'a>(path: &Cow<'a, BStr>) -> Option<Cow<'a, BStr>> { pub fn file_name<'a>(path: &Cow<'a, [u8]>) -> Option<Cow<'a, [u8]>> {
if path.is_empty() { if path.is_empty() {
return None; return None;
} else if path.last() == Some(b'.') { } else if path.last_byte() == Some(b'.') {
return None; return None;
} }
let last_slash = path.rfind_byte(b'/').map(|i| i + 1).unwrap_or(0); let last_slash = path.rfind_byte(b'/').map(|i| i + 1).unwrap_or(0);
@@ -39,7 +39,7 @@ pub fn file_name<'a>(path: &Cow<'a, BStr>) -> Option<Cow<'a, BStr>> {
/// a pattern like `*.rs` is obviously trying to match files with a `rs` /// a pattern like `*.rs` is obviously trying to match files with a `rs`
/// extension, but it also matches files like `.rs`, which doesn't have an /// extension, but it also matches files like `.rs`, which doesn't have an
/// extension according to std::path::Path::extension. /// extension according to std::path::Path::extension.
pub fn file_name_ext<'a>(name: &Cow<'a, BStr>) -> Option<Cow<'a, BStr>> { pub fn file_name_ext<'a>(name: &Cow<'a, [u8]>) -> Option<Cow<'a, [u8]>> {
if name.is_empty() { if name.is_empty() {
return None; return None;
} }
@@ -60,7 +60,7 @@ pub fn file_name_ext<'a>(name: &Cow<'a, BStr>) -> Option<Cow<'a, BStr>> {
/// Normalizes a path to use `/` as a separator everywhere, even on platforms /// Normalizes a path to use `/` as a separator everywhere, even on platforms
/// that recognize other characters as separators. /// that recognize other characters as separators.
#[cfg(unix)] #[cfg(unix)]
pub fn normalize_path(path: Cow<BStr>) -> Cow<BStr> { pub fn normalize_path(path: Cow<[u8]>) -> Cow<[u8]> {
// UNIX only uses /, so we're good. // UNIX only uses /, so we're good.
path path
} }
@@ -68,7 +68,7 @@ pub fn normalize_path(path: Cow<BStr>) -> Cow<BStr> {
/// Normalizes a path to use `/` as a separator everywhere, even on platforms /// Normalizes a path to use `/` as a separator everywhere, even on platforms
/// that recognize other characters as separators. /// that recognize other characters as separators.
#[cfg(not(unix))] #[cfg(not(unix))]
pub fn normalize_path(mut path: Cow<BStr>) -> Cow<BStr> { pub fn normalize_path(mut path: Cow<[u8]>) -> Cow<[u8]> {
use std::path::is_separator; use std::path::is_separator;
for i in 0..path.len() { for i in 0..path.len() {
@@ -84,7 +84,7 @@ pub fn normalize_path(mut path: Cow<BStr>) -> Cow<BStr> {
mod tests { mod tests {
use std::borrow::Cow; use std::borrow::Cow;
use bstr::{B, BString}; use bstr::{ByteVec, B};
use super::{file_name_ext, normalize_path}; use super::{file_name_ext, normalize_path};
@@ -92,7 +92,7 @@ mod tests {
($name:ident, $file_name:expr, $ext:expr) => { ($name:ident, $file_name:expr, $ext:expr) => {
#[test] #[test]
fn $name() { fn $name() {
let bs = BString::from($file_name); let bs = Vec::from($file_name);
let got = file_name_ext(&Cow::Owned(bs)); let got = file_name_ext(&Cow::Owned(bs));
assert_eq!($ext.map(|s| Cow::Borrowed(B(s))), got); assert_eq!($ext.map(|s| Cow::Borrowed(B(s))), got);
} }
@@ -109,7 +109,7 @@ mod tests {
($name:ident, $path:expr, $expected:expr) => { ($name:ident, $path:expr, $expected:expr) => {
#[test] #[test]
fn $name() { fn $name() {
let bs = BString::from_slice($path); let bs = Vec::from_slice($path);
let got = normalize_path(Cow::Owned(bs)); let got = normalize_path(Cow::Owned(bs));
assert_eq!($expected.to_vec(), got.into_owned()); assert_eq!($expected.to_vec(), got.into_owned());
} }

38
crates/globset/src/serde_impl.rs Normal file
View File

@@ -0,0 +1,38 @@
use serde::de::Error;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use Glob;
impl Serialize for Glob {
fn serialize<S: Serializer>(
&self,
serializer: S,
) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.glob())
}
}
impl<'de> Deserialize<'de> for Glob {
fn deserialize<D: Deserializer<'de>>(
deserializer: D,
) -> Result<Self, D::Error> {
let glob = <&str as Deserialize>::deserialize(deserializer)?;
Glob::new(glob).map_err(D::Error::custom)
}
}
#[cfg(test)]
mod tests {
use Glob;
#[test]
fn glob_json_works() {
let test_glob = Glob::new("src/**/*.rs").unwrap();
let ser = serde_json::to_string(&test_glob).unwrap();
assert_eq!(ser, "\"src/**/*.rs\"");
let de: Glob = serde_json::from_str(&ser).unwrap();
assert_eq!(test_glob, de);
}
}

0
grep/COPYING → crates/grep/COPYING
View File

18
grep/Cargo.toml → crates/grep/Cargo.toml
View File

@@ -1,24 +1,24 @@
[package] [package]
name = "grep" name = "grep"
version = "0.2.3" #:version version = "0.2.5" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"] authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """ description = """
Fast line oriented regex searching as a library. Fast line oriented regex searching as a library.
""" """
documentation = "http://burntsushi.net/rustdoc/grep/" documentation = "http://burntsushi.net/rustdoc/grep/"
homepage = "https://github.com/BurntSushi/ripgrep" homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/grep"
repository = "https://github.com/BurntSushi/ripgrep" repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/grep"
readme = "README.md" readme = "README.md"
keywords = ["regex", "grep", "egrep", "search", "pattern"] keywords = ["regex", "grep", "egrep", "search", "pattern"]
license = "Unlicense/MIT" license = "Unlicense/MIT"
[dependencies] [dependencies]
grep-cli = { version = "0.1.1", path = "../grep-cli" } grep-cli = { version = "0.1.4", path = "../cli" }
grep-matcher = { version = "0.1.2", path = "../grep-matcher" } grep-matcher = { version = "0.1.4", path = "../matcher" }
grep-pcre2 = { version = "0.1.3", path = "../grep-pcre2", optional = true } grep-pcre2 = { version = "0.1.4", path = "../pcre2", optional = true }
grep-printer = { version = "0.1.1", path = "../grep-printer" } grep-printer = { version = "0.1.4", path = "../printer" }
grep-regex = { version = "0.1.3", path = "../grep-regex" } grep-regex = { version = "0.1.6", path = "../regex" }
grep-searcher = { version = "0.1.1", path = "../grep-searcher" } grep-searcher = { version = "0.1.7", path = "../searcher" }
[dev-dependencies] [dev-dependencies]
termcolor = "1.0.4" termcolor = "1.0.4"

0
grep-matcher/LICENSE-MIT → crates/grep/LICENSE-MIT
View File

0
grep/README.md → crates/grep/README.md
View File

0
grep-matcher/UNLICENSE → crates/grep/UNLICENSE
View File

16
grep/examples/simplegrep.rs → crates/grep/examples/simplegrep.rs
View File

@@ -21,7 +21,7 @@ fn main() {
} }
} }
fn try_main() -> Result<(), Box<Error>> { fn try_main() -> Result<(), Box<dyn Error>> {
let mut args: Vec<OsString> = env::args_os().collect(); let mut args: Vec<OsString> = env::args_os().collect();
if args.len() < 2 { if args.len() < 2 {
return Err("Usage: simplegrep <pattern> [<path> ...]".into()); return Err("Usage: simplegrep <pattern> [<path> ...]".into());
@@ -32,7 +32,7 @@ fn try_main() -> Result<(), Box<Error>> {
search(cli::pattern_from_os(&args[1])?, &args[2..]) search(cli::pattern_from_os(&args[1])?, &args[2..])
} }
fn search(pattern: &str, paths: &[OsString]) -> Result<(), Box<Error>> { fn search(pattern: &str, paths: &[OsString]) -> Result<(), Box<dyn Error>> {
let matcher = RegexMatcher::new_line_matcher(&pattern)?; let matcher = RegexMatcher::new_line_matcher(&pattern)?;
let mut searcher = SearcherBuilder::new() let mut searcher = SearcherBuilder::new()
.binary_detection(BinaryDetection::quit(b'\x00')) .binary_detection(BinaryDetection::quit(b'\x00'))
@@ -40,13 +40,11 @@ fn search(pattern: &str, paths: &[OsString]) -> Result<(), Box<Error>> {
.build(); .build();
let mut printer = StandardBuilder::new() let mut printer = StandardBuilder::new()
.color_specs(ColorSpecs::default_with_color()) .color_specs(ColorSpecs::default_with_color())
.build(cli::stdout( .build(cli::stdout(if cli::is_tty_stdout() {
if cli::is_tty_stdout() { ColorChoice::Auto
ColorChoice::Auto } else {
} else { ColorChoice::Never
ColorChoice::Never }));
}
));
for path in paths { for path in paths {
for result in WalkDir::new(path) { for result in WalkDir::new(path) {

0
grep/src/lib.rs → crates/grep/src/lib.rs
View File

0
ignore/COPYING → crates/ignore/COPYING
View File

14
ignore/Cargo.toml → crates/ignore/Cargo.toml
View File

@@ -1,14 +1,14 @@
[package] [package]
name = "ignore" name = "ignore"
version = "0.4.7" #:version version = "0.4.12" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"] authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """ description = """
A fast library for efficiently matching ignore files such as `.gitignore` A fast library for efficiently matching ignore files such as `.gitignore`
against file paths. against file paths.
""" """
documentation = "https://docs.rs/ignore" documentation = "https://docs.rs/ignore"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/ignore" homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/ignore"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/ignore" repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/ignore"
readme = "README.md" readme = "README.md"
keywords = ["glob", "ignore", "gitignore", "pattern", "file"] keywords = ["glob", "ignore", "gitignore", "pattern", "file"]
license = "Unlicense/MIT" license = "Unlicense/MIT"
@@ -18,21 +18,19 @@ name = "ignore"
bench = false bench = false
[dependencies] [dependencies]
crossbeam-channel = "0.3.6" crossbeam-channel = "0.4.0"
crossbeam-utils = "0.7.0"
globset = { version = "0.4.3", path = "../globset" } globset = { version = "0.4.3", path = "../globset" }
lazy_static = "1.1" lazy_static = "1.1"
log = "0.4.5" log = "0.4.5"
memchr = "2.1" memchr = "2.1"
regex = "1.1" regex = "1.1"
same-file = "1.0.4" same-file = "1.0.4"
thread_local = "0.3.6" thread_local = "1"
walkdir = "2.2.7" walkdir = "2.2.7"
[target.'cfg(windows)'.dependencies.winapi-util] [target.'cfg(windows)'.dependencies.winapi-util]
version = "0.1.2" version = "0.1.2"
[dev-dependencies]
tempfile = "3.0.5"
[features] [features]
simd-accel = ["globset/simd-accel"] simd-accel = ["globset/simd-accel"]

0
grep-pcre2/LICENSE-MIT → crates/ignore/LICENSE-MIT
View File

0
ignore/README.md → crates/ignore/README.md
View File

0
grep-pcre2/UNLICENSE → crates/ignore/UNLICENSE
View File

0
ignore/examples/walk.rs → crates/ignore/examples/walk.rs
View File

246
crates/ignore/src/default_types.rs Normal file
View File

@@ -0,0 +1,246 @@
/// This list represents the default file types that ripgrep ships with. In
/// general, any file format is fair game, although it should generally be
/// limited to reasonably popular open formats. For other cases, you can add
/// types to each invocation of ripgrep with the '--type-add' flag.
///
/// If you would like to add or improve this list, please file a PR:
/// https://github.com/BurntSushi/ripgrep
///
/// Please try to keep this list sorted lexicographically and wrapped to 79
/// columns (inclusive).
#[rustfmt::skip]
pub const DEFAULT_TYPES: &[(&str, &[&str])] = &[
("agda", &["*.agda", "*.lagda"]),
("aidl", &["*.aidl"]),
("amake", &["*.mk", "*.bp"]),
("asciidoc", &["*.adoc", "*.asc", "*.asciidoc"]),
("asm", &["*.asm", "*.s", "*.S"]),
("asp", &[
"*.aspx", "*.aspx.cs", "*.aspx.cs", "*.ascx", "*.ascx.cs", "*.ascx.vb",
]),
("ats", &["*.ats", "*.dats", "*.sats", "*.hats"]),
("avro", &["*.avdl", "*.avpr", "*.avsc"]),
("awk", &["*.awk"]),
("bazel", &["*.bzl", "WORKSPACE", "BUILD", "BUILD.bazel"]),
("bitbake", &["*.bb", "*.bbappend", "*.bbclass", "*.conf", "*.inc"]),
("brotli", &["*.br"]),
("buildstream", &["*.bst"]),
("bzip2", &["*.bz2", "*.tbz2"]),
("c", &["*.[chH]", "*.[chH].in", "*.cats"]),
("cabal", &["*.cabal"]),
("cbor", &["*.cbor"]),
("ceylon", &["*.ceylon"]),
("clojure", &["*.clj", "*.cljc", "*.cljs", "*.cljx"]),
("cmake", &["*.cmake", "CMakeLists.txt"]),
("coffeescript", &["*.coffee"]),
("config", &["*.cfg", "*.conf", "*.config", "*.ini"]),
("coq", &["*.v"]),
("cpp", &[
"*.[ChH]", "*.cc", "*.[ch]pp", "*.[ch]xx", "*.hh", "*.inl",
"*.[ChH].in", "*.cc.in", "*.[ch]pp.in", "*.[ch]xx.in", "*.hh.in",
]),
("creole", &["*.creole"]),
("crystal", &["Projectfile", "*.cr"]),
("cs", &["*.cs"]),
("csharp", &["*.cs"]),
("cshtml", &["*.cshtml"]),
("css", &["*.css", "*.scss"]),
("csv", &["*.csv"]),
("cython", &["*.pyx", "*.pxi", "*.pxd"]),
("d", &["*.d"]),
("dart", &["*.dart"]),
("dhall", &["*.dhall"]),
("diff", &["*.patch", "*.diff"]),
("docker", &["*Dockerfile*"]),
("edn", &["*.edn"]),
("elisp", &["*.el"]),
("elixir", &["*.ex", "*.eex", "*.exs"]),
("elm", &["*.elm"]),
("erb", &["*.erb"]),
("erlang", &["*.erl", "*.hrl"]),
("fidl", &["*.fidl"]),
("fish", &["*.fish"]),
("fortran", &[
"*.f", "*.F", "*.f77", "*.F77", "*.pfo",
"*.f90", "*.F90", "*.f95", "*.F95",
]),
("fsharp", &["*.fs", "*.fsx", "*.fsi"]),
("gap", &["*.g", "*.gap", "*.gi", "*.gd", "*.tst"]),
("gn", &["*.gn", "*.gni"]),
("go", &["*.go"]),
("gradle", &["*.gradle"]),
("groovy", &["*.groovy", "*.gradle"]),
("gzip", &["*.gz", "*.tgz"]),
("h", &["*.h", "*.hpp"]),
("haml", &["*.haml"]),
("haskell", &["*.hs", "*.lhs", "*.cpphs", "*.c2hs", "*.hsc"]),
("hbs", &["*.hbs"]),
("hs", &["*.hs", "*.lhs"]),
("html", &["*.htm", "*.html", "*.ejs"]),
("idris", &["*.idr", "*.lidr"]),
("java", &["*.java", "*.jsp", "*.jspx", "*.properties"]),
("jinja", &["*.j2", "*.jinja", "*.jinja2"]),
("jl", &["*.jl"]),
("js", &["*.js", "*.jsx", "*.vue"]),
("json", &["*.json", "composer.lock"]),
("jsonl", &["*.jsonl"]),
("julia", &["*.jl"]),
("jupyter", &["*.ipynb", "*.jpynb"]),
("k", &["*.k"]),
("kotlin", &["*.kt", "*.kts"]),
("less", &["*.less"]),
("license", &[
// General
"COPYING", "COPYING[.-]*",
"COPYRIGHT", "COPYRIGHT[.-]*",
"EULA", "EULA[.-]*",
"licen[cs]e", "licen[cs]e.*",
"LICEN[CS]E", "LICEN[CS]E[.-]*", "*[.-]LICEN[CS]E*",
"NOTICE", "NOTICE[.-]*",
"PATENTS", "PATENTS[.-]*",
"UNLICEN[CS]E", "UNLICEN[CS]E[.-]*",
// GPL (gpl.txt, etc.)
"agpl[.-]*",
"gpl[.-]*",
"lgpl[.-]*",
// Other license-specific (APACHE-2.0.txt, etc.)
"AGPL-*[0-9]*",
"APACHE-*[0-9]*",
"BSD-*[0-9]*",
"CC-BY-*",
"GFDL-*[0-9]*",
"GNU-*[0-9]*",
"GPL-*[0-9]*",
"LGPL-*[0-9]*",
"MIT-*[0-9]*",
"MPL-*[0-9]*",
"OFL-*[0-9]*",
]),
("lisp", &["*.el", "*.jl", "*.lisp", "*.lsp", "*.sc", "*.scm"]),
("lock", &["*.lock", "package-lock.json"]),
("log", &["*.log"]),
("lua", &["*.lua"]),
("lz4", &["*.lz4"]),
("lzma", &["*.lzma"]),
("m4", &["*.ac", "*.m4"]),
("make", &[
"[Gg][Nn][Uu]makefile", "[Mm]akefile",
"[Gg][Nn][Uu]makefile.am", "[Mm]akefile.am",
"[Gg][Nn][Uu]makefile.in", "[Mm]akefile.in",
"*.mk", "*.mak"
]),
("mako", &["*.mako", "*.mao"]),
("man", &["*.[0-9lnpx]", "*.[0-9][cEFMmpSx]"]),
("markdown", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
("matlab", &["*.m"]),
("md", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
("mk", &["mkfile"]),
("ml", &["*.ml"]),
("msbuild", &[
"*.csproj", "*.fsproj", "*.vcxproj", "*.proj", "*.props", "*.targets",
]),
("nim", &["*.nim", "*.nimf", "*.nimble", "*.nims"]),
("nix", &["*.nix"]),
("objc", &["*.h", "*.m"]),
("objcpp", &["*.h", "*.mm"]),
("ocaml", &["*.ml", "*.mli", "*.mll", "*.mly"]),
("org", &["*.org", "*.org_archive"]),
("pascal", &["*.pas", "*.dpr", "*.lpr", "*.pp", "*.inc"]),
("pdf", &["*.pdf"]),
("perl", &["*.perl", "*.pl", "*.PL", "*.plh", "*.plx", "*.pm", "*.t"]),
("php", &["*.php", "*.php3", "*.php4", "*.php5", "*.phtml"]),
("pod", &["*.pod"]),
("postscript", &["*.eps", "*.ps"]),
("protobuf", &["*.proto"]),
("ps", &["*.cdxml", "*.ps1", "*.ps1xml", "*.psd1", "*.psm1"]),
("puppet", &["*.erb", "*.pp", "*.rb"]),
("purs", &["*.purs"]),
("py", &["*.py"]),
("qmake", &["*.pro", "*.pri", "*.prf"]),
("qml", &["*.qml"]),
("r", &["*.R", "*.r", "*.Rmd", "*.Rnw"]),
("rdoc", &["*.rdoc"]),
("readme", &["README*", "*README"]),
("robot", &["*.robot"]),
("rst", &["*.rst"]),
("ruby", &["Gemfile", "*.gemspec", ".irbrc", "Rakefile", "*.rb"]),
("rust", &["*.rs"]),
("sass", &["*.sass", "*.scss"]),
("scala", &["*.scala", "*.sbt"]),
("sh", &[
// Portable/misc. init files
".login", ".logout", ".profile", "profile",
// bash-specific init files
".bash_login", "bash_login",
".bash_logout", "bash_logout",
".bash_profile", "bash_profile",
".bashrc", "bashrc", "*.bashrc",
// csh-specific init files
".cshrc", "*.cshrc",
// ksh-specific init files
".kshrc", "*.kshrc",
// tcsh-specific init files
".tcshrc",
// zsh-specific init files
".zshenv", "zshenv",
".zlogin", "zlogin",
".zlogout", "zlogout",
".zprofile", "zprofile",
".zshrc", "zshrc",
// Extensions
"*.bash", "*.csh", "*.ksh", "*.sh", "*.tcsh", "*.zsh",
]),
("slim", &["*.skim", "*.slim", "*.slime"]),
("smarty", &["*.tpl"]),
("sml", &["*.sml", "*.sig"]),
("soy", &["*.soy"]),
("spark", &["*.spark"]),
("spec", &["*.spec"]),
("sql", &["*.sql", "*.psql"]),
("stylus", &["*.styl"]),
("sv", &["*.v", "*.vg", "*.sv", "*.svh", "*.h"]),
("svg", &["*.svg"]),
("swift", &["*.swift"]),
("swig", &["*.def", "*.i"]),
("systemd", &[
"*.automount", "*.conf", "*.device", "*.link", "*.mount", "*.path",
"*.scope", "*.service", "*.slice", "*.socket", "*.swap", "*.target",
"*.timer",
]),
("taskpaper", &["*.taskpaper"]),
("tcl", &["*.tcl"]),
("tex", &["*.tex", "*.ltx", "*.cls", "*.sty", "*.bib", "*.dtx", "*.ins"]),
("textile", &["*.textile"]),
("tf", &["*.tf"]),
("thrift", &["*.thrift"]),
("toml", &["*.toml", "Cargo.lock"]),
("ts", &["*.ts", "*.tsx"]),
("twig", &["*.twig"]),
("txt", &["*.txt"]),
("typoscript", &["*.typoscript", "*.ts"]),
("vala", &["*.vala"]),
("vb", &["*.vb"]),
("verilog", &["*.v", "*.vh", "*.sv", "*.svh"]),
("vhdl", &["*.vhd", "*.vhdl"]),
("vim", &["*.vim"]),
("vimscript", &["*.vim"]),
("webidl", &["*.idl", "*.webidl", "*.widl"]),
("wiki", &["*.mediawiki", "*.wiki"]),
("xml", &[
"*.xml", "*.xml.dist", "*.dtd", "*.xsl", "*.xslt", "*.xsd", "*.xjb",
"*.rng", "*.sch", "*.xhtml",
]),
("xz", &["*.xz", "*.txz"]),
("yacc", &["*.y"]),
("yaml", &["*.yaml", "*.yml"]),
("zig", &["*.zig"]),
("zsh", &[
".zshenv", "zshenv",
".zlogin", "zlogin",
".zlogout", "zlogout",
".zprofile", "zprofile",
".zshrc", "zshrc",
"*.zsh",
]),
("zstd", &["*.zst", "*.zstd"]),
];

481
ignore/src/dir.rs → crates/ignore/src/dir.rs
View File

@@ -14,13 +14,15 @@
// well. // well.
use std::collections::HashMap; use std::collections::HashMap;
use std::ffi::{OsString, OsStr}; use std::ffi::{OsStr, OsString};
use std::fs::{File, FileType};
use std::io::{self, BufRead};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use gitignore::{self, Gitignore, GitignoreBuilder}; use gitignore::{self, Gitignore, GitignoreBuilder};
use pathutil::{is_hidden, strip_prefix};
use overrides::{self, Override}; use overrides::{self, Override};
use pathutil::{is_hidden, strip_prefix};
use types::{self, Types}; use types::{self, Types};
use walk::DirEntry; use walk::DirEntry;
use {Error, Match, PartialErrorBuilder}; use {Error, Match, PartialErrorBuilder};
@@ -76,6 +78,9 @@ struct IgnoreOptions {
git_exclude: bool, git_exclude: bool,
/// Whether to ignore files case insensitively /// Whether to ignore files case insensitively
ignore_case_insensitive: bool, ignore_case_insensitive: bool,
/// Whether a git repository must be present in order to apply any
/// git-related ignore rules.
require_git: bool,
} }
/// Ignore is a matcher useful for recursively walking one or more directories. /// Ignore is a matcher useful for recursively walking one or more directories.
@@ -197,7 +202,11 @@ impl Ignore {
errs.maybe_push(err); errs.maybe_push(err);
igtmp.is_absolute_parent = true; igtmp.is_absolute_parent = true;
igtmp.absolute_base = Some(absolute_base.clone()); igtmp.absolute_base = Some(absolute_base.clone());
igtmp.has_git = parent.join(".git").exists(); igtmp.has_git = if self.0.opts.git_ignore {
parent.join(".git").exists()
} else {
false
};
ig = Ignore(Arc::new(igtmp)); ig = Ignore(Arc::new(igtmp));
compiled.insert(parent.as_os_str().to_os_string(), ig.clone()); compiled.insert(parent.as_os_str().to_os_string(), ig.clone());
} }
@@ -222,59 +231,70 @@ impl Ignore {
/// Like add_child, but takes a full path and returns an IgnoreInner. /// Like add_child, but takes a full path and returns an IgnoreInner.
fn add_child_path(&self, dir: &Path) -> (IgnoreInner, Option<Error>) { fn add_child_path(&self, dir: &Path) -> (IgnoreInner, Option<Error>) {
let git_type = if self.0.opts.git_ignore || self.0.opts.git_exclude {
dir.join(".git").metadata().ok().map(|md| md.file_type())
} else {
None
};
let has_git = git_type.map(|_| true).unwrap_or(false);
let mut errs = PartialErrorBuilder::default(); let mut errs = PartialErrorBuilder::default();
let custom_ig_matcher = let custom_ig_matcher = if self.0.custom_ignore_filenames.is_empty() {
if self.0.custom_ignore_filenames.is_empty() { Gitignore::empty()
Gitignore::empty() } else {
} else { let (m, err) = create_gitignore(
let (m, err) = &dir,
create_gitignore( &dir,
&self.0.custom_ignore_filenames,
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let ig_matcher = if !self.0.opts.ignore {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(
&dir,
&dir,
&[".ignore"],
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let gi_matcher = if !self.0.opts.git_ignore {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(
&dir,
&dir,
&[".gitignore"],
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let gi_exclude_matcher = if !self.0.opts.git_exclude {
Gitignore::empty()
} else {
match resolve_git_commondir(dir, git_type) {
Ok(git_dir) => {
let (m, err) = create_gitignore(
&dir, &dir,
&self.0.custom_ignore_filenames, &git_dir,
&["info/exclude"],
self.0.opts.ignore_case_insensitive, self.0.opts.ignore_case_insensitive,
); );
errs.maybe_push(err); errs.maybe_push(err);
m m
}; }
let ig_matcher = Err(err) => {
if !self.0.opts.ignore { errs.maybe_push(err);
Gitignore::empty() Gitignore::empty()
} else { }
let (m, err) = }
create_gitignore( };
&dir,
&[".ignore"],
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let gi_matcher =
if !self.0.opts.git_ignore {
Gitignore::empty()
} else {
let (m, err) =
create_gitignore(
&dir,
&[".gitignore"],
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let gi_exclude_matcher =
if !self.0.opts.git_exclude {
Gitignore::empty()
} else {
let (m, err) =
create_gitignore(
&dir,
&[".git/info/exclude"],
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let ig = IgnoreInner { let ig = IgnoreInner {
compiled: self.0.compiled.clone(), compiled: self.0.compiled.clone(),
dir: dir.to_path_buf(), dir: dir.to_path_buf(),
@@ -290,7 +310,7 @@ impl Ignore {
git_global_matcher: self.0.git_global_matcher.clone(), git_global_matcher: self.0.git_global_matcher.clone(),
git_ignore_matcher: gi_matcher, git_ignore_matcher: gi_matcher,
git_exclude_matcher: gi_exclude_matcher, git_exclude_matcher: gi_exclude_matcher,
has_git: dir.join(".git").exists(), has_git: has_git,
opts: self.0.opts, opts: self.0.opts,
}; };
(ig, errs.into_error_option()) (ig, errs.into_error_option())
@@ -299,12 +319,16 @@ impl Ignore {
/// Returns true if at least one type of ignore rule should be matched. /// Returns true if at least one type of ignore rule should be matched.
fn has_any_ignore_rules(&self) -> bool { fn has_any_ignore_rules(&self) -> bool {
let opts = self.0.opts; let opts = self.0.opts;
let has_custom_ignore_files = !self.0.custom_ignore_filenames.is_empty(); let has_custom_ignore_files =
!self.0.custom_ignore_filenames.is_empty();
let has_explicit_ignores = !self.0.explicit_ignores.is_empty(); let has_explicit_ignores = !self.0.explicit_ignores.is_empty();
opts.ignore || opts.git_global || opts.git_ignore opts.ignore
|| opts.git_exclude || has_custom_ignore_files || opts.git_global
|| has_explicit_ignores || opts.git_ignore
|| opts.git_exclude
|| has_custom_ignore_files
|| has_explicit_ignores
} }
/// Like `matched`, but works with a directory entry instead. /// Like `matched`, but works with a directory entry instead.
@@ -339,9 +363,11 @@ impl Ignore {
// return that result immediately. Overrides have the highest // return that result immediately. Overrides have the highest
// precedence. // precedence.
if !self.0.overrides.is_empty() { if !self.0.overrides.is_empty() {
let mat = let mat = self
self.0.overrides.matched(path, is_dir) .0
.map(IgnoreMatch::overrides); .overrides
.matched(path, is_dir)
.map(IgnoreMatch::overrides);
if !mat.is_none() { if !mat.is_none() {
return mat; return mat;
} }
@@ -374,56 +400,72 @@ impl Ignore {
path: &Path, path: &Path,
is_dir: bool, is_dir: bool,
) -> Match<IgnoreMatch<'a>> { ) -> Match<IgnoreMatch<'a>> {
let (mut m_custom_ignore, mut m_ignore, mut m_gi, mut m_gi_exclude, mut m_explicit) = let (
(Match::None, Match::None, Match::None, Match::None, Match::None); mut m_custom_ignore,
let any_git = self.parents().any(|ig| ig.0.has_git); mut m_ignore,
mut m_gi,
mut m_gi_exclude,
mut m_explicit,
) = (Match::None, Match::None, Match::None, Match::None, Match::None);
let any_git =
!self.0.opts.require_git || self.parents().any(|ig| ig.0.has_git);
let mut saw_git = false; let mut saw_git = false;
for ig in self.parents().take_while(|ig| !ig.0.is_absolute_parent) { for ig in self.parents().take_while(|ig| !ig.0.is_absolute_parent) {
if m_custom_ignore.is_none() { if m_custom_ignore.is_none() {
m_custom_ignore = m_custom_ignore =
ig.0.custom_ignore_matcher.matched(path, is_dir) ig.0.custom_ignore_matcher
.map(IgnoreMatch::gitignore); .matched(path, is_dir)
.map(IgnoreMatch::gitignore);
} }
if m_ignore.is_none() { if m_ignore.is_none() {
m_ignore = m_ignore =
ig.0.ignore_matcher.matched(path, is_dir) ig.0.ignore_matcher
.map(IgnoreMatch::gitignore); .matched(path, is_dir)
.map(IgnoreMatch::gitignore);
} }
if any_git && !saw_git && m_gi.is_none() { if any_git && !saw_git && m_gi.is_none() {
m_gi = m_gi =
ig.0.git_ignore_matcher.matched(path, is_dir) ig.0.git_ignore_matcher
.map(IgnoreMatch::gitignore); .matched(path, is_dir)
.map(IgnoreMatch::gitignore);
} }
if any_git && !saw_git && m_gi_exclude.is_none() { if any_git && !saw_git && m_gi_exclude.is_none() {
m_gi_exclude = m_gi_exclude =
ig.0.git_exclude_matcher.matched(path, is_dir) ig.0.git_exclude_matcher
.map(IgnoreMatch::gitignore); .matched(path, is_dir)
.map(IgnoreMatch::gitignore);
} }
saw_git = saw_git || ig.0.has_git; saw_git = saw_git || ig.0.has_git;
} }
if self.0.opts.parents { if self.0.opts.parents {
if let Some(abs_parent_path) = self.absolute_base() { if let Some(abs_parent_path) = self.absolute_base() {
let path = abs_parent_path.join(path); let path = abs_parent_path.join(path);
for ig in self.parents().skip_while(|ig|!ig.0.is_absolute_parent) { for ig in
self.parents().skip_while(|ig| !ig.0.is_absolute_parent)
{
if m_custom_ignore.is_none() { if m_custom_ignore.is_none() {
m_custom_ignore = m_custom_ignore =
ig.0.custom_ignore_matcher.matched(&path, is_dir) ig.0.custom_ignore_matcher
.map(IgnoreMatch::gitignore); .matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
} }
if m_ignore.is_none() { if m_ignore.is_none() {
m_ignore = m_ignore =
ig.0.ignore_matcher.matched(&path, is_dir) ig.0.ignore_matcher
.map(IgnoreMatch::gitignore); .matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
} }
if any_git && !saw_git && m_gi.is_none() { if any_git && !saw_git && m_gi.is_none() {
m_gi = m_gi =
ig.0.git_ignore_matcher.matched(&path, is_dir) ig.0.git_ignore_matcher
.map(IgnoreMatch::gitignore); .matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
} }
if any_git && !saw_git && m_gi_exclude.is_none() { if any_git && !saw_git && m_gi_exclude.is_none() {
m_gi_exclude = m_gi_exclude =
ig.0.git_exclude_matcher.matched(&path, is_dir) ig.0.git_exclude_matcher
.map(IgnoreMatch::gitignore); .matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
} }
saw_git = saw_git || ig.0.has_git; saw_git = saw_git || ig.0.has_git;
} }
@@ -435,16 +477,21 @@ impl Ignore {
} }
m_explicit = gi.matched(&path, is_dir).map(IgnoreMatch::gitignore); m_explicit = gi.matched(&path, is_dir).map(IgnoreMatch::gitignore);
} }
let m_global = let m_global = if any_git {
if any_git { self.0
self.0.git_global_matcher .git_global_matcher
.matched(&path, is_dir) .matched(&path, is_dir)
.map(IgnoreMatch::gitignore) .map(IgnoreMatch::gitignore)
} else { } else {
Match::None Match::None
}; };
m_custom_ignore.or(m_ignore).or(m_gi).or(m_gi_exclude).or(m_global).or(m_explicit) m_custom_ignore
.or(m_ignore)
.or(m_gi)
.or(m_gi_exclude)
.or(m_global)
.or(m_explicit)
} }
/// Returns an iterator over parent ignore matchers, including this one. /// Returns an iterator over parent ignore matchers, including this one.
@@ -515,6 +562,7 @@ impl IgnoreBuilder {
git_ignore: true, git_ignore: true,
git_exclude: true, git_exclude: true,
ignore_case_insensitive: false, ignore_case_insensitive: false,
require_git: true,
}, },
} }
} }
@@ -524,20 +572,19 @@ impl IgnoreBuilder {
/// The matcher returned won't match anything until ignore rules from /// The matcher returned won't match anything until ignore rules from
/// directories are added to it. /// directories are added to it.
pub fn build(&self) -> Ignore { pub fn build(&self) -> Ignore {
let git_global_matcher = let git_global_matcher = if !self.opts.git_global {
if !self.opts.git_global { Gitignore::empty()
Gitignore::empty() } else {
} else { let mut builder = GitignoreBuilder::new("");
let mut builder = GitignoreBuilder::new(""); builder
builder .case_insensitive(self.opts.ignore_case_insensitive)
.case_insensitive(self.opts.ignore_case_insensitive) .unwrap();
.unwrap(); let (gi, err) = builder.build_global();
let (gi, err) = builder.build_global(); if let Some(err) = err {
if let Some(err) = err { debug!("{}", err);
debug!("{}", err); }
} gi
gi };
};
Ignore(Arc::new(IgnoreInner { Ignore(Arc::new(IgnoreInner {
compiled: Arc::new(RwLock::new(HashMap::new())), compiled: Arc::new(RwLock::new(HashMap::new())),
@@ -548,7 +595,9 @@ impl IgnoreBuilder {
is_absolute_parent: true, is_absolute_parent: true,
absolute_base: None, absolute_base: None,
explicit_ignores: Arc::new(self.explicit_ignores.clone()), explicit_ignores: Arc::new(self.explicit_ignores.clone()),
custom_ignore_filenames: Arc::new(self.custom_ignore_filenames.clone()), custom_ignore_filenames: Arc::new(
self.custom_ignore_filenames.clone(),
),
custom_ignore_matcher: Gitignore::empty(), custom_ignore_matcher: Gitignore::empty(),
ignore_matcher: Gitignore::empty(), ignore_matcher: Gitignore::empty(),
git_global_matcher: Arc::new(git_global_matcher), git_global_matcher: Arc::new(git_global_matcher),
@@ -593,7 +642,7 @@ impl IgnoreBuilder {
/// later names. /// later names.
pub fn add_custom_ignore_filename<S: AsRef<OsStr>>( pub fn add_custom_ignore_filename<S: AsRef<OsStr>>(
&mut self, &mut self,
file_name: S file_name: S,
) -> &mut IgnoreBuilder { ) -> &mut IgnoreBuilder {
self.custom_ignore_filenames.push(file_name.as_ref().to_os_string()); self.custom_ignore_filenames.push(file_name.as_ref().to_os_string());
self self
@@ -664,6 +713,16 @@ impl IgnoreBuilder {
self self
} }
/// Whether a git repository is required to apply git-related ignore
/// rules (global rules, .gitignore and local exclude rules).
///
/// When disabled, git-related ignore rules are applied even when searching
/// outside a git repository.
pub fn require_git(&mut self, yes: bool) -> &mut IgnoreBuilder {
self.opts.require_git = yes;
self
}
/// Process ignore files case insensitively /// Process ignore files case insensitively
/// ///
/// This is disabled by default. /// This is disabled by default.
@@ -678,12 +737,15 @@ impl IgnoreBuilder {
/// Creates a new gitignore matcher for the directory given. /// Creates a new gitignore matcher for the directory given.
/// ///
/// Ignore globs are extracted from each of the file names in `dir` in the /// The matcher is meant to match files below `dir`.
/// order given (earlier names have lower precedence than later names). /// Ignore globs are extracted from each of the file names relative to
/// `dir_for_ignorefile` in the order given (earlier names have lower
/// precedence than later names).
/// ///
/// I/O errors are ignored. /// I/O errors are ignored.
pub fn create_gitignore<T: AsRef<OsStr>>( pub fn create_gitignore<T: AsRef<OsStr>>(
dir: &Path, dir: &Path,
dir_for_ignorefile: &Path,
names: &[T], names: &[T],
case_insensitive: bool, case_insensitive: bool,
) -> (Gitignore, Option<Error>) { ) -> (Gitignore, Option<Error>) {
@@ -691,8 +753,22 @@ pub fn create_gitignore<T: AsRef<OsStr>>(
let mut errs = PartialErrorBuilder::default(); let mut errs = PartialErrorBuilder::default();
builder.case_insensitive(case_insensitive).unwrap(); builder.case_insensitive(case_insensitive).unwrap();
for name in names { for name in names {
let gipath = dir.join(name.as_ref()); let gipath = dir_for_ignorefile.join(name.as_ref());
errs.maybe_push_ignore_io(builder.add(gipath)); // This check is not necessary, but is added for performance. Namely,
// a simple stat call checking for existence can often be just a bit
// quicker than actually trying to open a file. Since the number of
// directories without ignore files likely greatly exceeds the number
// with ignore files, this check generally makes sense.
//
// However, until demonstrated otherwise, we speculatively do not do
// this on Windows since Windows is notorious for having slow file
// system operations. Namely, it's not clear whether this analysis
// makes sense on Windows.
//
// For more details: https://github.com/BurntSushi/ripgrep/pull/1381
if cfg!(windows) || gipath.exists() {
errs.maybe_push_ignore_io(builder.add(gipath));
}
} }
let gi = match builder.build() { let gi = match builder.build() {
Ok(gi) => gi, Ok(gi) => gi,
@@ -704,16 +780,71 @@ pub fn create_gitignore<T: AsRef<OsStr>>(
(gi, errs.into_error_option()) (gi, errs.into_error_option())
} }
/// Find the GIT_COMMON_DIR for the given git worktree.
///
/// This is the directory that may contain a private ignore file
/// "info/exclude". Unlike git, this function does *not* read environment
/// variables GIT_DIR and GIT_COMMON_DIR, because it is not clear how to use
/// them when multiple repositories are searched.
///
/// Some I/O errors are ignored.
fn resolve_git_commondir(
dir: &Path,
git_type: Option<FileType>,
) -> Result<PathBuf, Option<Error>> {
let git_dir_path = || dir.join(".git");
let git_dir = git_dir_path();
if !git_type.map_or(false, |ft| ft.is_file()) {
return Ok(git_dir);
}
let file = match File::open(git_dir) {
Ok(file) => io::BufReader::new(file),
Err(err) => {
return Err(Some(Error::Io(err).with_path(git_dir_path())));
}
};
let dot_git_line = match file.lines().next() {
Some(Ok(line)) => line,
Some(Err(err)) => {
return Err(Some(Error::Io(err).with_path(git_dir_path())));
}
None => return Err(None),
};
if !dot_git_line.starts_with("gitdir: ") {
return Err(None);
}
let real_git_dir = PathBuf::from(&dot_git_line["gitdir: ".len()..]);
let git_commondir_file = || real_git_dir.join("commondir");
let file = match File::open(git_commondir_file()) {
Ok(file) => io::BufReader::new(file),
Err(err) => {
return Err(Some(Error::Io(err).with_path(git_commondir_file())));
}
};
let commondir_line = match file.lines().next() {
Some(Ok(line)) => line,
Some(Err(err)) => {
return Err(Some(Error::Io(err).with_path(git_commondir_file())));
}
None => return Err(None),
};
let commondir_abs = if commondir_line.starts_with(".") {
real_git_dir.join(commondir_line) // relative commondir
} else {
PathBuf::from(commondir_line)
};
Ok(commondir_abs)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::fs::{self, File}; use std::fs::{self, File};
use std::io::Write; use std::io::{self, Write};
use std::path::Path; use std::path::Path;
use tempfile::{self, TempDir};
use dir::IgnoreBuilder; use dir::IgnoreBuilder;
use gitignore::Gitignore; use gitignore::Gitignore;
use tests::TempDir;
use Error; use Error;
fn wfile<P: AsRef<Path>>(path: P, contents: &str) { fn wfile<P: AsRef<Path>>(path: P, contents: &str) {
@@ -732,19 +863,19 @@ mod tests {
} }
} }
fn tmpdir(prefix: &str) -> TempDir { fn tmpdir() -> TempDir {
tempfile::Builder::new().prefix(prefix).tempdir().unwrap() TempDir::new().unwrap()
} }
#[test] #[test]
fn explicit_ignore() { fn explicit_ignore() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
wfile(td.path().join("not-an-ignore"), "foo\n!bar"); wfile(td.path().join("not-an-ignore"), "foo\n!bar");
let (gi, err) = Gitignore::new(td.path().join("not-an-ignore")); let (gi, err) = Gitignore::new(td.path().join("not-an-ignore"));
assert!(err.is_none()); assert!(err.is_none());
let (ig, err) = IgnoreBuilder::new() let (ig, err) =
.add_ignore(gi).build().add_child(td.path()); IgnoreBuilder::new().add_ignore(gi).build().add_child(td.path());
assert!(err.is_none()); assert!(err.is_none());
assert!(ig.matched("foo", false).is_ignore()); assert!(ig.matched("foo", false).is_ignore());
assert!(ig.matched("bar", false).is_whitelist()); assert!(ig.matched("bar", false).is_whitelist());
@@ -753,7 +884,7 @@ mod tests {
#[test] #[test]
fn git_exclude() { fn git_exclude() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
mkdirp(td.path().join(".git/info")); mkdirp(td.path().join(".git/info"));
wfile(td.path().join(".git/info/exclude"), "foo\n!bar"); wfile(td.path().join(".git/info/exclude"), "foo\n!bar");
@@ -766,7 +897,7 @@ mod tests {
#[test] #[test]
fn gitignore() { fn gitignore() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
mkdirp(td.path().join(".git")); mkdirp(td.path().join(".git"));
wfile(td.path().join(".gitignore"), "foo\n!bar"); wfile(td.path().join(".gitignore"), "foo\n!bar");
@@ -779,7 +910,7 @@ mod tests {
#[test] #[test]
fn gitignore_no_git() { fn gitignore_no_git() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
wfile(td.path().join(".gitignore"), "foo\n!bar"); wfile(td.path().join(".gitignore"), "foo\n!bar");
let (ig, err) = IgnoreBuilder::new().build().add_child(td.path()); let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());
@@ -789,9 +920,24 @@ mod tests {
assert!(ig.matched("baz", false).is_none()); assert!(ig.matched("baz", false).is_none());
} }
#[test]
fn gitignore_allowed_no_git() {
let td = tmpdir();
wfile(td.path().join(".gitignore"), "foo\n!bar");
let (ig, err) = IgnoreBuilder::new()
.require_git(false)
.build()
.add_child(td.path());
assert!(err.is_none());
assert!(ig.matched("foo", false).is_ignore());
assert!(ig.matched("bar", false).is_whitelist());
assert!(ig.matched("baz", false).is_none());
}
#[test] #[test]
fn ignore() { fn ignore() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
wfile(td.path().join(".ignore"), "foo\n!bar"); wfile(td.path().join(".ignore"), "foo\n!bar");
let (ig, err) = IgnoreBuilder::new().build().add_child(td.path()); let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());
@@ -803,13 +949,14 @@ mod tests {
#[test] #[test]
fn custom_ignore() { fn custom_ignore() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
let custom_ignore = ".customignore"; let custom_ignore = ".customignore";
wfile(td.path().join(custom_ignore), "foo\n!bar"); wfile(td.path().join(custom_ignore), "foo\n!bar");
let (ig, err) = IgnoreBuilder::new() let (ig, err) = IgnoreBuilder::new()
.add_custom_ignore_filename(custom_ignore) .add_custom_ignore_filename(custom_ignore)
.build().add_child(td.path()); .build()
.add_child(td.path());
assert!(err.is_none()); assert!(err.is_none());
assert!(ig.matched("foo", false).is_ignore()); assert!(ig.matched("foo", false).is_ignore());
assert!(ig.matched("bar", false).is_whitelist()); assert!(ig.matched("bar", false).is_whitelist());
@@ -819,14 +966,15 @@ mod tests {
// Tests that a custom ignore file will override an .ignore. // Tests that a custom ignore file will override an .ignore.
#[test] #[test]
fn custom_ignore_over_ignore() { fn custom_ignore_over_ignore() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
let custom_ignore = ".customignore"; let custom_ignore = ".customignore";
wfile(td.path().join(".ignore"), "foo"); wfile(td.path().join(".ignore"), "foo");
wfile(td.path().join(custom_ignore), "!foo"); wfile(td.path().join(custom_ignore), "!foo");
let (ig, err) = IgnoreBuilder::new() let (ig, err) = IgnoreBuilder::new()
.add_custom_ignore_filename(custom_ignore) .add_custom_ignore_filename(custom_ignore)
.build().add_child(td.path()); .build()
.add_child(td.path());
assert!(err.is_none()); assert!(err.is_none());
assert!(ig.matched("foo", false).is_whitelist()); assert!(ig.matched("foo", false).is_whitelist());
} }
@@ -834,7 +982,7 @@ mod tests {
// Tests that earlier custom ignore files have lower precedence than later. // Tests that earlier custom ignore files have lower precedence than later.
#[test] #[test]
fn custom_ignore_precedence() { fn custom_ignore_precedence() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
let custom_ignore1 = ".customignore1"; let custom_ignore1 = ".customignore1";
let custom_ignore2 = ".customignore2"; let custom_ignore2 = ".customignore2";
wfile(td.path().join(custom_ignore1), "foo"); wfile(td.path().join(custom_ignore1), "foo");
@@ -843,7 +991,8 @@ mod tests {
let (ig, err) = IgnoreBuilder::new() let (ig, err) = IgnoreBuilder::new()
.add_custom_ignore_filename(custom_ignore1) .add_custom_ignore_filename(custom_ignore1)
.add_custom_ignore_filename(custom_ignore2) .add_custom_ignore_filename(custom_ignore2)
.build().add_child(td.path()); .build()
.add_child(td.path());
assert!(err.is_none()); assert!(err.is_none());
assert!(ig.matched("foo", false).is_whitelist()); assert!(ig.matched("foo", false).is_whitelist());
} }
@@ -851,7 +1000,7 @@ mod tests {
// Tests that an .ignore will override a .gitignore. // Tests that an .ignore will override a .gitignore.
#[test] #[test]
fn ignore_over_gitignore() { fn ignore_over_gitignore() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
wfile(td.path().join(".gitignore"), "foo"); wfile(td.path().join(".gitignore"), "foo");
wfile(td.path().join(".ignore"), "!foo"); wfile(td.path().join(".ignore"), "!foo");
@@ -863,7 +1012,7 @@ mod tests {
// Tests that exclude has lower precedent than both .ignore and .gitignore. // Tests that exclude has lower precedent than both .ignore and .gitignore.
#[test] #[test]
fn exclude_lowest() { fn exclude_lowest() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
wfile(td.path().join(".gitignore"), "!foo"); wfile(td.path().join(".gitignore"), "!foo");
wfile(td.path().join(".ignore"), "!bar"); wfile(td.path().join(".ignore"), "!bar");
mkdirp(td.path().join(".git/info")); mkdirp(td.path().join(".git/info"));
@@ -878,7 +1027,7 @@ mod tests {
#[test] #[test]
fn errored() { fn errored() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
wfile(td.path().join(".gitignore"), "{foo"); wfile(td.path().join(".gitignore"), "{foo");
let (_, err) = IgnoreBuilder::new().build().add_child(td.path()); let (_, err) = IgnoreBuilder::new().build().add_child(td.path());
@@ -887,7 +1036,7 @@ mod tests {
#[test] #[test]
fn errored_both() { fn errored_both() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
wfile(td.path().join(".gitignore"), "{foo"); wfile(td.path().join(".gitignore"), "{foo");
wfile(td.path().join(".ignore"), "{bar"); wfile(td.path().join(".ignore"), "{bar");
@@ -897,7 +1046,7 @@ mod tests {
#[test] #[test]
fn errored_partial() { fn errored_partial() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
mkdirp(td.path().join(".git")); mkdirp(td.path().join(".git"));
wfile(td.path().join(".gitignore"), "{foo\nbar"); wfile(td.path().join(".gitignore"), "{foo\nbar");
@@ -908,7 +1057,7 @@ mod tests {
#[test] #[test]
fn errored_partial_and_ignore() { fn errored_partial_and_ignore() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
wfile(td.path().join(".gitignore"), "{foo\nbar"); wfile(td.path().join(".gitignore"), "{foo\nbar");
wfile(td.path().join(".ignore"), "!bar"); wfile(td.path().join(".ignore"), "!bar");
@@ -919,7 +1068,7 @@ mod tests {
#[test] #[test]
fn not_present_empty() { fn not_present_empty() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
let (_, err) = IgnoreBuilder::new().build().add_child(td.path()); let (_, err) = IgnoreBuilder::new().build().add_child(td.path());
assert!(err.is_none()); assert!(err.is_none());
@@ -929,7 +1078,7 @@ mod tests {
fn stops_at_git_dir() { fn stops_at_git_dir() {
// This tests that .gitignore files beyond a .git barrier aren't // This tests that .gitignore files beyond a .git barrier aren't
// matched, but .ignore files are. // matched, but .ignore files are.
let td = tmpdir("ignore-test-"); let td = tmpdir();
mkdirp(td.path().join(".git")); mkdirp(td.path().join(".git"));
mkdirp(td.path().join("foo/.git")); mkdirp(td.path().join("foo/.git"));
wfile(td.path().join(".gitignore"), "foo"); wfile(td.path().join(".gitignore"), "foo");
@@ -950,7 +1099,7 @@ mod tests {
#[test] #[test]
fn absolute_parent() { fn absolute_parent() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
mkdirp(td.path().join(".git")); mkdirp(td.path().join(".git"));
mkdirp(td.path().join("foo")); mkdirp(td.path().join("foo"));
wfile(td.path().join(".gitignore"), "bar"); wfile(td.path().join(".gitignore"), "bar");
@@ -973,7 +1122,7 @@ mod tests {
#[test] #[test]
fn absolute_parent_anchored() { fn absolute_parent_anchored() {
let td = tmpdir("ignore-test-"); let td = tmpdir();
mkdirp(td.path().join(".git")); mkdirp(td.path().join(".git"));
mkdirp(td.path().join("src/llvm")); mkdirp(td.path().join("src/llvm"));
wfile(td.path().join(".gitignore"), "/llvm/\nfoo"); wfile(td.path().join(".gitignore"), "/llvm/\nfoo");
@@ -990,4 +1139,62 @@ mod tests {
assert!(ig2.matched("foo", false).is_ignore()); assert!(ig2.matched("foo", false).is_ignore());
assert!(ig2.matched("src/foo", false).is_ignore()); assert!(ig2.matched("src/foo", false).is_ignore());
} }
#[test]
fn git_info_exclude_in_linked_worktree() {
let td = tmpdir();
let git_dir = td.path().join(".git");
mkdirp(git_dir.join("info"));
wfile(git_dir.join("info/exclude"), "ignore_me");
mkdirp(git_dir.join("worktrees/linked-worktree"));
let commondir_path =
|| git_dir.join("worktrees/linked-worktree/commondir");
mkdirp(td.path().join("linked-worktree"));
let worktree_git_dir_abs = format!(
"gitdir: {}",
git_dir.join("worktrees/linked-worktree").to_str().unwrap(),
);
wfile(td.path().join("linked-worktree/.git"), &worktree_git_dir_abs);
// relative commondir
wfile(commondir_path(), "../..");
let ib = IgnoreBuilder::new().build();
let (ignore, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_none());
assert!(ignore.matched("ignore_me", false).is_ignore());
// absolute commondir
wfile(commondir_path(), git_dir.to_str().unwrap());
let (ignore, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_none());
assert!(ignore.matched("ignore_me", false).is_ignore());
// missing commondir file
assert!(fs::remove_file(commondir_path()).is_ok());
let (_, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_some());
assert!(match err {
Some(Error::WithPath { path, err }) => {
if path != commondir_path() {
false
} else {
match *err {
Error::Io(ioerr) => {
ioerr.kind() == io::ErrorKind::NotFound
}
_ => false,
}
}
}
_ => false,
});
wfile(td.path().join("linked-worktree/.git"), "garbage");
let (_, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_none());
wfile(td.path().join("linked-worktree/.git"), "gitdir: garbage");
let (_, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_some());
}
} }

61
ignore/src/gitignore.rs → crates/ignore/src/gitignore.rs
View File

@@ -249,7 +249,7 @@ impl Gitignore {
return Match::None; return Match::None;
} }
let path = path.as_ref(); let path = path.as_ref();
let _matches = self.matches.as_ref().unwrap().get_default(); let _matches = self.matches.as_ref().unwrap().get_or_default();
let mut matches = _matches.borrow_mut(); let mut matches = _matches.borrow_mut();
let candidate = Candidate::new(path); let candidate = Candidate::new(path);
self.set.matches_candidate_into(&candidate, &mut *matches); self.set.matches_candidate_into(&candidate, &mut *matches);
@@ -332,13 +332,10 @@ impl GitignoreBuilder {
pub fn build(&self) -> Result<Gitignore, Error> { pub fn build(&self) -> Result<Gitignore, Error> {
let nignore = self.globs.iter().filter(|g| !g.is_whitelist()).count(); let nignore = self.globs.iter().filter(|g| !g.is_whitelist()).count();
let nwhite = self.globs.iter().filter(|g| g.is_whitelist()).count(); let nwhite = self.globs.iter().filter(|g| g.is_whitelist()).count();
let set = let set = self
self.builder.build().map_err(|err| { .builder
Error::Glob { .build()
glob: None, .map_err(|err| Error::Glob { glob: None, err: err.to_string() })?;
err: err.to_string(),
}
})?;
Ok(Gitignore { Ok(Gitignore {
set: set, set: set,
root: self.root.clone(), root: self.root.clone(),
@@ -499,18 +496,15 @@ impl GitignoreBuilder {
if glob.actual.ends_with("/**") { if glob.actual.ends_with("/**") {
glob.actual = format!("{}/*", glob.actual); glob.actual = format!("{}/*", glob.actual);
} }
let parsed = let parsed = GlobBuilder::new(&glob.actual)
GlobBuilder::new(&glob.actual) .literal_separator(true)
.literal_separator(true) .case_insensitive(self.case_insensitive)
.case_insensitive(self.case_insensitive) .backslash_escape(true)
.backslash_escape(true) .build()
.build() .map_err(|err| Error::Glob {
.map_err(|err| { glob: Some(glob.original.clone()),
Error::Glob { err: err.kind().to_string(),
glob: Some(glob.original.clone()), })?;
err: err.kind().to_string(),
}
})?;
self.builder.add(parsed); self.builder.add(parsed);
self.globs.push(glob); self.globs.push(glob);
Ok(self) Ok(self)
@@ -537,7 +531,7 @@ impl GitignoreBuilder {
/// ///
/// Note that the file path returned may not exist. /// Note that the file path returned may not exist.
fn gitconfig_excludes_path() -> Option<PathBuf> { fn gitconfig_excludes_path() -> Option<PathBuf> {
// git supports $HOME/.gitconfig and $XDG_CONFIG_DIR/git/config. Notably, // git supports $HOME/.gitconfig and $XDG_CONFIG_HOME/git/config. Notably,
// both can be active at the same time, where $HOME/.gitconfig takes // both can be active at the same time, where $HOME/.gitconfig takes
// precedent. So if $HOME/.gitconfig defines a `core.excludesFile`, then // precedent. So if $HOME/.gitconfig defines a `core.excludesFile`, then
// we're done. // we're done.
@@ -568,7 +562,7 @@ fn gitconfig_home_contents() -> Option<Vec<u8>> {
} }
/// Returns the file contents of git's global config file, if one exists, in /// Returns the file contents of git's global config file, if one exists, in
/// the user's XDG_CONFIG_DIR directory. /// the user's XDG_CONFIG_HOME directory.
fn gitconfig_xdg_contents() -> Option<Vec<u8>> { fn gitconfig_xdg_contents() -> Option<Vec<u8>> {
let path = env::var_os("XDG_CONFIG_HOME") let path = env::var_os("XDG_CONFIG_HOME")
.and_then(|x| if x.is_empty() { None } else { Some(PathBuf::from(x)) }) .and_then(|x| if x.is_empty() { None } else { Some(PathBuf::from(x)) })
@@ -599,9 +593,8 @@ fn parse_excludes_file(data: &[u8]) -> Option<PathBuf> {
// probably works in more circumstances. I guess we would ideally have // probably works in more circumstances. I guess we would ideally have
// a full INI parser. Yuck. // a full INI parser. Yuck.
lazy_static! { lazy_static! {
static ref RE: Regex = Regex::new( static ref RE: Regex =
r"(?im)^\s*excludesfile\s*=\s*(.+)\s*$" Regex::new(r"(?im)^\s*excludesfile\s*=\s*(.+)\s*$").unwrap();
).unwrap();
}; };
let caps = match RE.captures(data) { let caps = match RE.captures(data) {
None => return None, None => return None,
@@ -630,8 +623,8 @@ fn home_dir() -> Option<PathBuf> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::path::Path;
use super::{Gitignore, GitignoreBuilder}; use super::{Gitignore, GitignoreBuilder};
use std::path::Path;
fn gi_from_str<P: AsRef<Path>>(root: P, s: &str) -> Gitignore { fn gi_from_str<P: AsRef<Path>>(root: P, s: &str) -> Gitignore {
let mut builder = GitignoreBuilder::new(root); let mut builder = GitignoreBuilder::new(root);
@@ -726,8 +719,11 @@ mod tests {
not_ignored!(ignot12, ROOT, "\n\n\n", "foo"); not_ignored!(ignot12, ROOT, "\n\n\n", "foo");
not_ignored!(ignot13, ROOT, "foo/**", "foo", true); not_ignored!(ignot13, ROOT, "foo/**", "foo", true);
not_ignored!( not_ignored!(
ignot14, "./third_party/protobuf", "m4/ltoptions.m4", ignot14,
"./third_party/protobuf/csharp/src/packages/repositories.config"); "./third_party/protobuf",
"m4/ltoptions.m4",
"./third_party/protobuf/csharp/src/packages/repositories.config"
);
not_ignored!(ignot15, ROOT, "!/bar", "foo/bar"); not_ignored!(ignot15, ROOT, "!/bar", "foo/bar");
not_ignored!(ignot16, ROOT, "*\n!**/", "foo", true); not_ignored!(ignot16, ROOT, "*\n!**/", "foo", true);
not_ignored!(ignot17, ROOT, "src/*.rs", "src/grep/src/main.rs"); not_ignored!(ignot17, ROOT, "src/*.rs", "src/grep/src/main.rs");
@@ -771,9 +767,12 @@ mod tests {
#[test] #[test]
fn case_insensitive() { fn case_insensitive() {
let gi = GitignoreBuilder::new(ROOT) let gi = GitignoreBuilder::new(ROOT)
.case_insensitive(true).unwrap() .case_insensitive(true)
.add_str(None, "*.html").unwrap() .unwrap()
.build().unwrap(); .add_str(None, "*.html")
.unwrap()
.build()
.unwrap();
assert!(gi.matched("foo.html", false).is_ignore()); assert!(gi.matched("foo.html", false).is_ignore());
assert!(gi.matched("foo.HTML", false).is_ignore()); assert!(gi.matched("foo.HTML", false).is_ignore());
assert!(!gi.matched("foo.htm", false).is_ignore()); assert!(!gi.matched("foo.htm", false).is_ignore());

131
ignore/src/lib.rs → crates/ignore/src/lib.rs
View File

@@ -55,8 +55,6 @@ extern crate log;
extern crate memchr; extern crate memchr;
extern crate regex; extern crate regex;
extern crate same_file; extern crate same_file;
#[cfg(test)]
extern crate tempfile;
extern crate thread_local; extern crate thread_local;
extern crate walkdir; extern crate walkdir;
#[cfg(windows)] #[cfg(windows)]
@@ -67,12 +65,16 @@ use std::fmt;
use std::io; use std::io;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
pub use walk::{DirEntry, Walk, WalkBuilder, WalkParallel, WalkState}; pub use walk::{
DirEntry, ParallelVisitor, ParallelVisitorBuilder, Walk, WalkBuilder,
WalkParallel, WalkState,
};
mod default_types;
mod dir; mod dir;
pub mod gitignore; pub mod gitignore;
mod pathutil;
pub mod overrides; pub mod overrides;
mod pathutil;
pub mod types; pub mod types;
mod walk; mod walk;
@@ -145,20 +147,14 @@ impl Clone for Error {
Error::WithDepth { depth, ref err } => { Error::WithDepth { depth, ref err } => {
Error::WithDepth { depth: depth, err: err.clone() } Error::WithDepth { depth: depth, err: err.clone() }
} }
Error::Loop { ref ancestor, ref child } => { Error::Loop { ref ancestor, ref child } => Error::Loop {
Error::Loop { ancestor: ancestor.clone(),
ancestor: ancestor.clone(), child: child.clone(),
child: child.clone() },
} Error::Io(ref err) => match err.raw_os_error() {
} Some(e) => Error::Io(io::Error::from_raw_os_error(e)),
Error::Io(ref err) => { None => Error::Io(io::Error::new(err.kind(), err.to_string())),
match err.raw_os_error() { },
Some(e) => Error::Io(io::Error::from_raw_os_error(e)),
None => {
Error::Io(io::Error::new(err.kind(), err.to_string()))
}
}
}
Error::Glob { ref glob, ref err } => { Error::Glob { ref glob, ref err } => {
Error::Glob { glob: glob.clone(), err: err.clone() } Error::Glob { glob: glob.clone(), err: err.clone() }
} }
@@ -221,19 +217,14 @@ impl Error {
/// Turn an error into a tagged error with the given depth. /// Turn an error into a tagged error with the given depth.
fn with_depth(self, depth: usize) -> Error { fn with_depth(self, depth: usize) -> Error {
Error::WithDepth { Error::WithDepth { depth: depth, err: Box::new(self) }
depth: depth,
err: Box::new(self),
}
} }
/// Turn an error into a tagged error with the given file path and line /// Turn an error into a tagged error with the given file path and line
/// number. If path is empty, then it is omitted from the error. /// number. If path is empty, then it is omitted from the error.
fn tagged<P: AsRef<Path>>(self, path: P, lineno: u64) -> Error { fn tagged<P: AsRef<Path>>(self, path: P, lineno: u64) -> Error {
let errline = Error::WithLineNumber { let errline =
line: lineno, Error::WithLineNumber { line: lineno, err: Box::new(self) };
err: Box::new(self),
};
if path.as_ref().as_os_str().is_empty() { if path.as_ref().as_os_str().is_empty() {
return errline; return errline;
} }
@@ -255,16 +246,14 @@ impl Error {
let path = err.path().map(|p| p.to_path_buf()); let path = err.path().map(|p| p.to_path_buf());
let mut ig_err = Error::Io(io::Error::from(err)); let mut ig_err = Error::Io(io::Error::from(err));
if let Some(path) = path { if let Some(path) = path {
ig_err = Error::WithPath { ig_err = Error::WithPath { path: path, err: Box::new(ig_err) };
path: path,
err: Box::new(ig_err),
};
} }
ig_err ig_err
} }
} }
impl error::Error for Error { impl error::Error for Error {
#[allow(deprecated)]
fn description(&self) -> &str { fn description(&self) -> &str {
match *self { match *self {
Error::Partial(_) => "partial error", Error::Partial(_) => "partial error",
@@ -295,11 +284,13 @@ impl fmt::Display for Error {
write!(f, "{}: {}", path.display(), err) write!(f, "{}: {}", path.display(), err)
} }
Error::WithDepth { ref err, .. } => err.fmt(f), Error::WithDepth { ref err, .. } => err.fmt(f),
Error::Loop { ref ancestor, ref child } => { Error::Loop { ref ancestor, ref child } => write!(
write!(f, "File system loop found: \ f,
"File system loop found: \
{} points to an ancestor {}", {} points to an ancestor {}",
child.display(), ancestor.display()) child.display(),
} ancestor.display()
),
Error::Io(ref err) => err.fmt(f), Error::Io(ref err) => err.fmt(f),
Error::Glob { glob: None, ref err } => write!(f, "{}", err), Error::Glob { glob: None, ref err } => write!(f, "{}", err),
Error::Glob { glob: Some(ref glob), ref err } => { Error::Glob { glob: Some(ref glob), ref err } => {
@@ -308,10 +299,11 @@ impl fmt::Display for Error {
Error::UnrecognizedFileType(ref ty) => { Error::UnrecognizedFileType(ref ty) => {
write!(f, "unrecognized file type: {}", ty) write!(f, "unrecognized file type: {}", ty)
} }
Error::InvalidDefinition => { Error::InvalidDefinition => write!(
write!(f, "invalid definition (format is type:glob, e.g., \ f,
html:*.html)") "invalid definition (format is type:glob, e.g., \
} html:*.html)"
),
} }
} }
} }
@@ -442,3 +434,66 @@ impl<T> Match<T> {
} }
} }
} }
#[cfg(test)]
mod tests {
use std::env;
use std::error;
use std::fs;
use std::path::{Path, PathBuf};
use std::result;
/// A convenient result type alias.
pub type Result<T> =
result::Result<T, Box<dyn error::Error + Send + Sync>>;
macro_rules! err {
($($tt:tt)*) => {
Box::<dyn error::Error + Send + Sync>::from(format!($($tt)*))
}
}
/// A simple wrapper for creating a temporary directory that is
/// automatically deleted when it's dropped.
///
/// We use this in lieu of tempfile because tempfile brings in too many
/// dependencies.
#[derive(Debug)]
pub struct TempDir(PathBuf);
impl Drop for TempDir {
fn drop(&mut self) {
fs::remove_dir_all(&self.0).unwrap();
}
}
impl TempDir {
/// Create a new empty temporary directory under the system's configured
/// temporary directory.
pub fn new() -> Result<TempDir> {
use std::sync::atomic::{AtomicUsize, Ordering};
static TRIES: usize = 100;
static COUNTER: AtomicUsize = AtomicUsize::new(0);
let tmpdir = env::temp_dir();
for _ in 0..TRIES {
let count = COUNTER.fetch_add(1, Ordering::SeqCst);
let path = tmpdir.join("rust-ignore").join(count.to_string());
if path.is_dir() {
continue;
}
fs::create_dir_all(&path).map_err(|e| {
err!("failed to create {}: {}", path.display(), e)
})?;
return Ok(TempDir(path));
}
Err(err!("failed to create temp dir after {} tries", TRIES))
}
/// Return the underlying path to this temporary directory.
pub fn path(&self) -> &Path {
&self.0
}
}
}

18
ignore/src/overrides.rs → crates/ignore/src/overrides.rs
View File

@@ -115,9 +115,7 @@ impl OverrideBuilder {
/// ///
/// Matching is done relative to the directory path provided. /// Matching is done relative to the directory path provided.
pub fn new<P: AsRef<Path>>(path: P) -> OverrideBuilder { pub fn new<P: AsRef<Path>>(path: P) -> OverrideBuilder {
OverrideBuilder { OverrideBuilder { builder: GitignoreBuilder::new(path) }
builder: GitignoreBuilder::new(path),
}
} }
/// Builds a new override matcher from the globs added so far. /// Builds a new override matcher from the globs added so far.
@@ -240,9 +238,12 @@ mod tests {
#[test] #[test]
fn case_insensitive() { fn case_insensitive() {
let ov = OverrideBuilder::new(ROOT) let ov = OverrideBuilder::new(ROOT)
.case_insensitive(true).unwrap() .case_insensitive(true)
.add("*.html").unwrap() .unwrap()
.build().unwrap(); .add("*.html")
.unwrap()
.build()
.unwrap();
assert!(ov.matched("foo.html", false).is_whitelist()); assert!(ov.matched("foo.html", false).is_whitelist());
assert!(ov.matched("foo.HTML", false).is_whitelist()); assert!(ov.matched("foo.HTML", false).is_whitelist());
assert!(ov.matched("foo.htm", false).is_ignore()); assert!(ov.matched("foo.htm", false).is_ignore());
@@ -251,9 +252,8 @@ mod tests {
#[test] #[test]
fn default_case_sensitive() { fn default_case_sensitive() {
let ov = OverrideBuilder::new(ROOT) let ov =
.add("*.html").unwrap() OverrideBuilder::new(ROOT).add("*.html").unwrap().build().unwrap();
.build().unwrap();
assert!(ov.matched("foo.html", false).is_whitelist()); assert!(ov.matched("foo.html", false).is_whitelist());
assert!(ov.matched("foo.HTML", false).is_ignore()); assert!(ov.matched("foo.HTML", false).is_ignore());
assert!(ov.matched("foo.htm", false).is_ignore()); assert!(ov.matched("foo.htm", false).is_ignore());

4
ignore/src/pathutil.rs → crates/ignore/src/pathutil.rs
View File

@@ -91,8 +91,8 @@ pub fn strip_prefix<'a, P: AsRef<Path> + ?Sized>(
/// the empty string. /// the empty string.
#[cfg(unix)] #[cfg(unix)]
pub fn is_file_name<P: AsRef<Path>>(path: P) -> bool { pub fn is_file_name<P: AsRef<Path>>(path: P) -> bool {
use std::os::unix::ffi::OsStrExt;
use memchr::memchr; use memchr::memchr;
use std::os::unix::ffi::OsStrExt;
let path = path.as_ref().as_os_str().as_bytes(); let path = path.as_ref().as_os_str().as_bytes();
memchr(b'/', path).is_none() memchr(b'/', path).is_none()
@@ -113,8 +113,8 @@ pub fn is_file_name<P: AsRef<Path>>(path: P) -> bool {
pub fn file_name<'a, P: AsRef<Path> + ?Sized>( pub fn file_name<'a, P: AsRef<Path> + ?Sized>(
path: &'a P, path: &'a P,
) -> Option<&'a OsStr> { ) -> Option<&'a OsStr> {
use std::os::unix::ffi::OsStrExt;
use memchr::memrchr; use memchr::memrchr;
use std::os::unix::ffi::OsStrExt;
let path = path.as_ref().as_os_str().as_bytes(); let path = path.as_ref().as_os_str().as_bytes();
if path.is_empty() { if path.is_empty() {

280
ignore/src/types.rs → crates/ignore/src/types.rs
View File

@@ -93,233 +93,10 @@ use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
use regex::Regex; use regex::Regex;
use thread_local::ThreadLocal; use thread_local::ThreadLocal;
use default_types::DEFAULT_TYPES;
use pathutil::file_name; use pathutil::file_name;
use {Error, Match}; use {Error, Match};
const DEFAULT_TYPES: &'static [(&'static str, &'static [&'static str])] = &[
("agda", &["*.agda", "*.lagda"]),
("ats", &["*.ats", "*.dats", "*.sats", "*.hats"]),
("aidl", &["*.aidl"]),
("amake", &["*.mk", "*.bp"]),
("asciidoc", &["*.adoc", "*.asc", "*.asciidoc"]),
("asm", &["*.asm", "*.s", "*.S"]),
("asp", &["*.aspx", "*.aspx.cs", "*.aspx.cs", "*.ascx", "*.ascx.cs", "*.ascx.vb"]),
("avro", &["*.avdl", "*.avpr", "*.avsc"]),
("awk", &["*.awk"]),
("bazel", &["*.bzl", "WORKSPACE", "BUILD", "BUILD.bazel"]),
("bitbake", &["*.bb", "*.bbappend", "*.bbclass", "*.conf", "*.inc"]),
("brotli", &["*.br"]),
("buildstream", &["*.bst"]),
("bzip2", &["*.bz2", "*.tbz2"]),
("c", &["*.[chH]", "*.[chH].in", "*.cats"]),
("cabal", &["*.cabal"]),
("cbor", &["*.cbor"]),
("ceylon", &["*.ceylon"]),
("clojure", &["*.clj", "*.cljc", "*.cljs", "*.cljx"]),
("cmake", &["*.cmake", "CMakeLists.txt"]),
("coffeescript", &["*.coffee"]),
("creole", &["*.creole"]),
("config", &["*.cfg", "*.conf", "*.config", "*.ini"]),
("cpp", &[
"*.[ChH]", "*.cc", "*.[ch]pp", "*.[ch]xx", "*.hh", "*.inl",
"*.[ChH].in", "*.cc.in", "*.[ch]pp.in", "*.[ch]xx.in", "*.hh.in",
]),
("crystal", &["Projectfile", "*.cr"]),
("cs", &["*.cs"]),
("csharp", &["*.cs"]),
("cshtml", &["*.cshtml"]),
("css", &["*.css", "*.scss"]),
("csv", &["*.csv"]),
("cython", &["*.pyx", "*.pxi", "*.pxd"]),
("dart", &["*.dart"]),
("d", &["*.d"]),
("dhall", &["*.dhall"]),
("docker", &["*Dockerfile*"]),
("elisp", &["*.el"]),
("elixir", &["*.ex", "*.eex", "*.exs"]),
("elm", &["*.elm"]),
("erlang", &["*.erl", "*.hrl"]),
("fidl", &["*.fidl"]),
("fish", &["*.fish"]),
("fortran", &[
"*.f", "*.F", "*.f77", "*.F77", "*.pfo",
"*.f90", "*.F90", "*.f95", "*.F95",
]),
("fsharp", &["*.fs", "*.fsx", "*.fsi"]),
("gn", &["*.gn", "*.gni"]),
("go", &["*.go"]),
("gzip", &["*.gz", "*.tgz"]),
("groovy", &["*.groovy", "*.gradle"]),
("h", &["*.h", "*.hpp"]),
("hbs", &["*.hbs"]),
("haskell", &["*.hs", "*.lhs", "*.cpphs", "*.c2hs", "*.hsc"]),
("hs", &["*.hs", "*.lhs"]),
("html", &["*.htm", "*.html", "*.ejs"]),
("idris", &["*.idr", "*.lidr"]),
("java", &["*.java", "*.jsp", "*.jspx", "*.properties"]),
("jinja", &["*.j2", "*.jinja", "*.jinja2"]),
("js", &[
"*.js", "*.jsx", "*.vue",
]),
("json", &["*.json", "composer.lock"]),
("jsonl", &["*.jsonl"]),
("julia", &["*.jl"]),
("jupyter", &["*.ipynb", "*.jpynb"]),
("jl", &["*.jl"]),
("kotlin", &["*.kt", "*.kts"]),
("less", &["*.less"]),
("license", &[
// General
"COPYING", "COPYING[.-]*",
"COPYRIGHT", "COPYRIGHT[.-]*",
"EULA", "EULA[.-]*",
"licen[cs]e", "licen[cs]e.*",
"LICEN[CS]E", "LICEN[CS]E[.-]*", "*[.-]LICEN[CS]E*",
"NOTICE", "NOTICE[.-]*",
"PATENTS", "PATENTS[.-]*",
"UNLICEN[CS]E", "UNLICEN[CS]E[.-]*",
// GPL (gpl.txt, etc.)
"agpl[.-]*",
"gpl[.-]*",
"lgpl[.-]*",
// Other license-specific (APACHE-2.0.txt, etc.)
"AGPL-*[0-9]*",
"APACHE-*[0-9]*",
"BSD-*[0-9]*",
"CC-BY-*",
"GFDL-*[0-9]*",
"GNU-*[0-9]*",
"GPL-*[0-9]*",
"LGPL-*[0-9]*",
"MIT-*[0-9]*",
"MPL-*[0-9]*",
"OFL-*[0-9]*",
]),
("lisp", &["*.el", "*.jl", "*.lisp", "*.lsp", "*.sc", "*.scm"]),
("lock", &["*.lock", "package-lock.json"]),
("log", &["*.log"]),
("lua", &["*.lua"]),
("lzma", &["*.lzma"]),
("lz4", &["*.lz4"]),
("m4", &["*.ac", "*.m4"]),
("make", &[
"[Gg][Nn][Uu]makefile", "[Mm]akefile",
"[Gg][Nn][Uu]makefile.am", "[Mm]akefile.am",
"[Gg][Nn][Uu]makefile.in", "[Mm]akefile.in",
"*.mk", "*.mak"
]),
("mako", &["*.mako", "*.mao"]),
("markdown", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
("md", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
("man", &["*.[0-9lnpx]", "*.[0-9][cEFMmpSx]"]),
("matlab", &["*.m"]),
("mk", &["mkfile"]),
("ml", &["*.ml"]),
("msbuild", &[
"*.csproj", "*.fsproj", "*.vcxproj", "*.proj", "*.props", "*.targets"
]),
("nim", &["*.nim"]),
("nix", &["*.nix"]),
("objc", &["*.h", "*.m"]),
("objcpp", &["*.h", "*.mm"]),
("ocaml", &["*.ml", "*.mli", "*.mll", "*.mly"]),
("org", &["*.org"]),
("pascal", &["*.pas", "*.dpr", "*.lpr", "*.pp", "*.inc"]),
("perl", &["*.perl", "*.pl", "*.PL", "*.plh", "*.plx", "*.pm", "*.t"]),
("pdf", &["*.pdf"]),
("php", &["*.php", "*.php3", "*.php4", "*.php5", "*.phtml"]),
("pod", &["*.pod"]),
("postscript", &[".eps", ".ps"]),
("protobuf", &["*.proto"]),
("ps", &["*.cdxml", "*.ps1", "*.ps1xml", "*.psd1", "*.psm1"]),
("puppet", &["*.erb", "*.pp", "*.rb"]),
("purs", &["*.purs"]),
("py", &["*.py"]),
("qmake", &["*.pro", "*.pri", "*.prf"]),
("qml", &["*.qml"]),
("readme", &["README*", "*README"]),
("r", &["*.R", "*.r", "*.Rmd", "*.Rnw"]),
("rdoc", &["*.rdoc"]),
("rst", &["*.rst"]),
("ruby", &["Gemfile", "*.gemspec", ".irbrc", "Rakefile", "*.rb"]),
("rust", &["*.rs"]),
("sass", &["*.sass", "*.scss"]),
("scala", &["*.scala", "*.sbt"]),
("sh", &[
// Portable/misc. init files
".login", ".logout", ".profile", "profile",
// bash-specific init files
".bash_login", "bash_login",
".bash_logout", "bash_logout",
".bash_profile", "bash_profile",
".bashrc", "bashrc", "*.bashrc",
// csh-specific init files
".cshrc", "*.cshrc",
// ksh-specific init files
".kshrc", "*.kshrc",
// tcsh-specific init files
".tcshrc",
// zsh-specific init files
".zshenv", "zshenv",
".zlogin", "zlogin",
".zlogout", "zlogout",
".zprofile", "zprofile",
".zshrc", "zshrc",
// Extensions
"*.bash", "*.csh", "*.ksh", "*.sh", "*.tcsh", "*.zsh",
]),
("smarty", &["*.tpl"]),
("sml", &["*.sml", "*.sig"]),
("soy", &["*.soy"]),
("spark", &["*.spark"]),
("sql", &["*.sql", "*.psql"]),
("stylus", &["*.styl"]),
("sv", &["*.v", "*.vg", "*.sv", "*.svh", "*.h"]),
("svg", &["*.svg"]),
("swift", &["*.swift"]),
("swig", &["*.def", "*.i"]),
("systemd", &[
"*.automount", "*.conf", "*.device", "*.link", "*.mount", "*.path",
"*.scope", "*.service", "*.slice", "*.socket", "*.swap", "*.target",
"*.timer",
]),
("taskpaper", &["*.taskpaper"]),
("tcl", &["*.tcl"]),
("tex", &["*.tex", "*.ltx", "*.cls", "*.sty", "*.bib", "*.dtx", "*.ins"]),
("textile", &["*.textile"]),
("thrift", &["*.thrift"]),
("tf", &["*.tf"]),
("ts", &["*.ts", "*.tsx"]),
("txt", &["*.txt"]),
("toml", &["*.toml", "Cargo.lock"]),
("twig", &["*.twig"]),
("vala", &["*.vala"]),
("vb", &["*.vb"]),
("verilog", &["*.v", "*.vh", "*.sv", "*.svh"]),
("vhdl", &["*.vhd", "*.vhdl"]),
("vim", &["*.vim"]),
("vimscript", &["*.vim"]),
("wiki", &["*.mediawiki", "*.wiki"]),
("webidl", &["*.idl", "*.webidl", "*.widl"]),
("xml", &[
"*.xml", "*.xml.dist", "*.dtd", "*.xsl", "*.xslt", "*.xsd", "*.xjb",
"*.rng", "*.sch",
]),
("xz", &["*.xz", "*.txz"]),
("yacc", &["*.y"]),
("yaml", &["*.yaml", "*.yml"]),
("zig", &["*.zig"]),
("zsh", &[
".zshenv", "zshenv",
".zlogin", "zlogin",
".zlogout", "zlogout",
".zprofile", "zprofile",
".zshrc", "zshrc",
"*.zsh",
]),
("zstd", &["*.zst", "*.zstd"]),
];
/// Glob represents a single glob in a set of file type definitions. /// Glob represents a single glob in a set of file type definitions.
/// ///
/// There may be more than one glob for a particular file type. /// There may be more than one glob for a particular file type.
@@ -349,7 +126,7 @@ enum GlobInner<'a> {
which: usize, which: usize,
/// Whether the selection was negated or not. /// Whether the selection was negated or not.
negated: bool, negated: bool,
} },
} }
impl<'a> Glob<'a> { impl<'a> Glob<'a> {
@@ -363,9 +140,7 @@ impl<'a> Glob<'a> {
pub fn file_type_def(&self) -> Option<&FileTypeDef> { pub fn file_type_def(&self) -> Option<&FileTypeDef> {
match self { match self {
Glob(GlobInner::UnmatchedIgnore) => None, Glob(GlobInner::UnmatchedIgnore) => None,
Glob(GlobInner::Matched { def, .. }) => { Glob(GlobInner::Matched { def, .. }) => Some(def),
Some(def)
},
} }
} }
} }
@@ -512,7 +287,7 @@ impl Types {
return Match::None; return Match::None;
} }
}; };
let mut matches = self.matches.get_default().borrow_mut(); let mut matches = self.matches.get_or_default().borrow_mut();
self.set.matches_into(name, &mut *matches); self.set.matches_into(name, &mut *matches);
// The highest precedent match is the last one. // The highest precedent match is the last one.
if let Some(&i) = matches.last() { if let Some(&i) = matches.last() {
@@ -551,10 +326,7 @@ impl TypesBuilder {
/// of default type definitions can be added with `add_defaults`, and /// of default type definitions can be added with `add_defaults`, and
/// additional type definitions can be added with `select` and `negate`. /// additional type definitions can be added with `select` and `negate`.
pub fn new() -> TypesBuilder { pub fn new() -> TypesBuilder {
TypesBuilder { TypesBuilder { types: HashMap::new(), selections: vec![] }
types: HashMap::new(),
selections: vec![],
}
} }
/// Build the current set of file type definitions *and* selections into /// Build the current set of file type definitions *and* selections into
@@ -579,19 +351,18 @@ impl TypesBuilder {
GlobBuilder::new(glob) GlobBuilder::new(glob)
.literal_separator(true) .literal_separator(true)
.build() .build()
.map_err(|err| { .map_err(|err| Error::Glob {
Error::Glob { glob: Some(glob.to_string()),
glob: Some(glob.to_string()), err: err.kind().to_string(),
err: err.kind().to_string(), })?,
} );
})?);
glob_to_selection.push((isel, iglob)); glob_to_selection.push((isel, iglob));
} }
selections.push(selection.clone().map(move |_| def)); selections.push(selection.clone().map(move |_| def));
} }
let set = build_set.build().map_err(|err| { let set = build_set
Error::Glob { glob: None, err: err.to_string() } .build()
})?; .map_err(|err| Error::Glob { glob: None, err: err.to_string() })?;
Ok(Types { Ok(Types {
defs: defs, defs: defs,
selections: selections, selections: selections,
@@ -663,9 +434,14 @@ impl TypesBuilder {
return Err(Error::InvalidDefinition); return Err(Error::InvalidDefinition);
} }
let (key, glob) = (name.to_string(), glob.to_string()); let (key, glob) = (name.to_string(), glob.to_string());
self.types.entry(key).or_insert_with(|| { self.types
FileTypeDef { name: name.to_string(), globs: vec![] } .entry(key)
}).globs.push(glob); .or_insert_with(|| FileTypeDef {
name: name.to_string(),
globs: vec![],
})
.globs
.push(glob);
Ok(()) Ok(())
} }
@@ -692,7 +468,10 @@ impl TypesBuilder {
3 => { 3 => {
let name = parts[0]; let name = parts[0];
let types_string = parts[2]; let types_string = parts[2];
if name.is_empty() || parts[1] != "include" || types_string.is_empty() { if name.is_empty()
|| parts[1] != "include"
|| types_string.is_empty()
{
return Err(Error::InvalidDefinition); return Err(Error::InvalidDefinition);
} }
let types = types_string.split(','); let types = types_string.split(',');
@@ -702,14 +481,15 @@ impl TypesBuilder {
return Err(Error::InvalidDefinition); return Err(Error::InvalidDefinition);
} }
for type_name in types { for type_name in types {
let globs = self.types.get(type_name).unwrap().globs.clone(); let globs =
self.types.get(type_name).unwrap().globs.clone();
for glob in globs { for glob in globs {
self.add(name, &glob)?; self.add(name, &glob)?;
} }
} }
Ok(()) Ok(())
} }
_ => Err(Error::InvalidDefinition) _ => Err(Error::InvalidDefinition),
} }
} }
@@ -766,7 +546,7 @@ mod tests {
"rust:*.rs", "rust:*.rs",
"js:*.js", "js:*.js",
"foo:*.{rs,foo}", "foo:*.{rs,foo}",
"combo:include:html,rust" "combo:include:html,rust",
] ]
} }
@@ -800,7 +580,7 @@ mod tests {
"combo:include:html,python", "combo:include:html,python",
// Bad format // Bad format
"combo:foobar:html,rust", "combo:foobar:html,rust",
"" "",
]; ];
for def in bad_defs { for def in bad_defs {
assert!(btypes.add_def(def).is_err()); assert!(btypes.add_def(def).is_err());

904
ignore/src/walk.rs → crates/ignore/src/walk.rs
View File

File diff suppressed because it is too large Load Diff

0
ignore/tests/gitignore_matched_path_or_any_parents_tests.gitignore → crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.gitignore
View File

43
ignore/tests/gitignore_matched_path_or_any_parents_tests.rs → crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.rs
View File

@@ -55,7 +55,6 @@ fn test_files_in_root() {
assert!(m("ROOT/file_root_33").is_none()); assert!(m("ROOT/file_root_33").is_none());
} }
#[test] #[test]
fn test_files_in_deep() { fn test_files_in_deep() {
let gitignore = get_gitignore(); let gitignore = get_gitignore();
@@ -88,7 +87,6 @@ fn test_files_in_deep() {
assert!(m("ROOT/parent_dir/file_deep_33").is_none()); assert!(m("ROOT/parent_dir/file_deep_33").is_none());
} }
#[test] #[test]
fn test_dirs_in_root() { fn test_dirs_in_root() {
let gitignore = get_gitignore(); let gitignore = get_gitignore();
@@ -193,7 +191,6 @@ fn test_dirs_in_root() {
assert!(m("ROOT/dir_root_33/child_dir/file", false).is_ignore()); assert!(m("ROOT/dir_root_33/child_dir/file", false).is_ignore());
} }
#[test] #[test]
fn test_dirs_in_deep() { fn test_dirs_in_deep() {
let gitignore = get_gitignore(); let gitignore = get_gitignore();
@@ -205,17 +202,13 @@ fn test_dirs_in_deep() {
assert!(m("ROOT/parent_dir/dir_deep_00", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_00", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_00/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_00/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_00/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_00/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_00/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_00/child_dir/file", false).is_ignore()
);
// 01 // 01
assert!(m("ROOT/parent_dir/dir_deep_01", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_01", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_01/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_01/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_01/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_01/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_01/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_01/child_dir/file", false).is_ignore()
);
// 02 // 02
assert!(m("ROOT/parent_dir/dir_deep_02", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_02", true).is_none());
@@ -257,67 +250,51 @@ fn test_dirs_in_deep() {
assert!(m("ROOT/parent_dir/dir_deep_20", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_20", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_20/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_20/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_20/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_20/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_20/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_20/child_dir/file", false).is_ignore()
);
// 21 // 21
assert!(m("ROOT/parent_dir/dir_deep_21", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_21", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_21/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_21/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_21/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_21/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_21/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_21/child_dir/file", false).is_ignore()
);
// 22 // 22
// dir itself doesn't match // dir itself doesn't match
assert!(m("ROOT/parent_dir/dir_deep_22", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_22", true).is_none());
assert!(m("ROOT/parent_dir/dir_deep_22/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_22/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_22/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_22/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_22/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_22/child_dir/file", false).is_ignore()
);
// 23 // 23
// dir itself doesn't match // dir itself doesn't match
assert!(m("ROOT/parent_dir/dir_deep_23", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_23", true).is_none());
assert!(m("ROOT/parent_dir/dir_deep_23/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_23/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_23/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_23/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_23/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_23/child_dir/file", false).is_ignore()
);
// 30 // 30
assert!(m("ROOT/parent_dir/dir_deep_30", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_30", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_30/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_30/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_30/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_30/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_30/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_30/child_dir/file", false).is_ignore()
);
// 31 // 31
assert!(m("ROOT/parent_dir/dir_deep_31", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_31", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_31/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_31/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_31/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_31/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_31/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_31/child_dir/file", false).is_ignore()
);
// 32 // 32
// dir itself doesn't match // dir itself doesn't match
assert!(m("ROOT/parent_dir/dir_deep_32", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_32", true).is_none());
assert!(m("ROOT/parent_dir/dir_deep_32/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_32/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_32/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_32/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_32/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_32/child_dir/file", false).is_ignore()
);
// 33 // 33
// dir itself doesn't match // dir itself doesn't match
assert!(m("ROOT/parent_dir/dir_deep_33", true).is_none()); assert!(m("ROOT/parent_dir/dir_deep_33", true).is_none());
assert!(m("ROOT/parent_dir/dir_deep_33/file", false).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_33/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_33/child_dir", true).is_ignore()); assert!(m("ROOT/parent_dir/dir_deep_33/child_dir", true).is_ignore());
assert!( assert!(m("ROOT/parent_dir/dir_deep_33/child_dir/file", false).is_ignore());
m("ROOT/parent_dir/dir_deep_33/child_dir/file", false).is_ignore()
);
} }

6
grep-matcher/Cargo.toml → crates/matcher/Cargo.toml
View File

@@ -1,13 +1,13 @@
[package] [package]
name = "grep-matcher" name = "grep-matcher"
version = "0.1.2" #:version version = "0.1.4" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"] authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """ description = """
A trait for regular expressions, with a focus on line oriented search. A trait for regular expressions, with a focus on line oriented search.
""" """
documentation = "https://docs.rs/grep-matcher" documentation = "https://docs.rs/grep-matcher"
homepage = "https://github.com/BurntSushi/ripgrep" homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/matcher"
repository = "https://github.com/BurntSushi/ripgrep" repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/matcher"
readme = "README.md" readme = "README.md"
keywords = ["regex", "pattern", "trait"] keywords = ["regex", "pattern", "trait"]
license = "Unlicense/MIT" license = "Unlicense/MIT"

0
grep-printer/LICENSE-MIT → crates/matcher/LICENSE-MIT
View File

0
grep-matcher/README.md → crates/matcher/README.md
View File

0
grep-printer/UNLICENSE → crates/matcher/UNLICENSE
View File

8
grep-matcher/src/interpolate.rs → crates/matcher/src/interpolate.rs
View File

@@ -19,7 +19,7 @@ pub fn interpolate<A, N>(
dst: &mut Vec<u8>, dst: &mut Vec<u8>,
) where ) where
A: FnMut(usize, &mut Vec<u8>), A: FnMut(usize, &mut Vec<u8>),
N: FnMut(&str) -> Option<usize> N: FnMut(&str) -> Option<usize>,
{ {
while !replacement.is_empty() { while !replacement.is_empty() {
match memchr(b'$', replacement) { match memchr(b'$', replacement) {
@@ -134,14 +134,14 @@ fn find_cap_ref(replacement: &[u8]) -> Option<CaptureRef> {
/// Returns true if and only if the given byte is allowed in a capture name. /// Returns true if and only if the given byte is allowed in a capture name.
fn is_valid_cap_letter(b: &u8) -> bool { fn is_valid_cap_letter(b: &u8) -> bool {
match *b { match *b {
b'0' ... b'9' | b'a' ... b'z' | b'A' ... b'Z' | b'_' => true, b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' | b'_' => true,
_ => false, _ => false,
} }
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{CaptureRef, find_cap_ref, interpolate}; use super::{find_cap_ref, interpolate, CaptureRef};
macro_rules! find { macro_rules! find {
($name:ident, $text:expr) => { ($name:ident, $text:expr) => {
@@ -211,7 +211,7 @@ mod tests {
fn $name() { fn $name() {
assert_eq!($expected, interpolate_string($map, $caps, $hay)); assert_eq!($expected, interpolate_string($map, $caps, $hay));
} }
} };
} }
interp!( interp!(

69
grep-matcher/src/lib.rs → crates/matcher/src/lib.rs
View File

@@ -278,7 +278,7 @@ impl LineTerminator {
} }
} }
impl Default for LineTerminator { impl Default for LineTerminator {
#[inline] #[inline]
fn default() -> LineTerminator { fn default() -> LineTerminator {
LineTerminator::byte(b'\n') LineTerminator::byte(b'\n')
@@ -439,7 +439,8 @@ pub trait Captures {
haystack: &[u8], haystack: &[u8],
replacement: &[u8], replacement: &[u8],
dst: &mut Vec<u8>, dst: &mut Vec<u8>,
) where F: FnMut(&str) -> Option<usize> ) where
F: FnMut(&str) -> Option<usize>,
{ {
interpolate( interpolate(
replacement, replacement,
@@ -463,12 +464,18 @@ pub struct NoCaptures(());
impl NoCaptures { impl NoCaptures {
/// Create an empty set of capturing groups. /// Create an empty set of capturing groups.
pub fn new() -> NoCaptures { NoCaptures(()) } pub fn new() -> NoCaptures {
NoCaptures(())
}
} }
impl Captures for NoCaptures { impl Captures for NoCaptures {
fn len(&self) -> usize { 0 } fn len(&self) -> usize {
fn get(&self, _: usize) -> Option<Match> { None } 0
}
fn get(&self, _: usize) -> Option<Match> {
None
}
} }
/// NoError provides an error type for matchers that never produce errors. /// NoError provides an error type for matchers that never produce errors.
@@ -481,7 +488,9 @@ impl Captures for NoCaptures {
pub struct NoError(()); pub struct NoError(());
impl ::std::error::Error for NoError { impl ::std::error::Error for NoError {
fn description(&self) -> &str { "no error" } fn description(&self) -> &str {
"no error"
}
} }
impl fmt::Display for NoError { impl fmt::Display for NoError {
@@ -599,10 +608,7 @@ pub trait Matcher {
/// ///
/// The text encoding of `haystack` is not strictly specified. Matchers are /// The text encoding of `haystack` is not strictly specified. Matchers are
/// advised to assume UTF-8, or at worst, some ASCII compatible encoding. /// advised to assume UTF-8, or at worst, some ASCII compatible encoding.
fn find( fn find(&self, haystack: &[u8]) -> Result<Option<Match>, Self::Error> {
&self,
haystack: &[u8],
) -> Result<Option<Match>, Self::Error> {
self.find_at(haystack, 0) self.find_at(haystack, 0)
} }
@@ -614,7 +620,8 @@ pub trait Matcher {
haystack: &[u8], haystack: &[u8],
mut matched: F, mut matched: F,
) -> Result<(), Self::Error> ) -> Result<(), Self::Error>
where F: FnMut(Match) -> bool where
F: FnMut(Match) -> bool,
{ {
self.try_find_iter(haystack, |m| Ok(matched(m))) self.try_find_iter(haystack, |m| Ok(matched(m)))
.map(|r: Result<(), ()>| r.unwrap()) .map(|r: Result<(), ()>| r.unwrap())
@@ -632,7 +639,8 @@ pub trait Matcher {
haystack: &[u8], haystack: &[u8],
mut matched: F, mut matched: F,
) -> Result<Result<(), E>, Self::Error> ) -> Result<Result<(), E>, Self::Error>
where F: FnMut(Match) -> Result<bool, E> where
F: FnMut(Match) -> Result<bool, E>,
{ {
let mut last_end = 0; let mut last_end = 0;
let mut last_match = None; let mut last_match = None;
@@ -690,7 +698,8 @@ pub trait Matcher {
caps: &mut Self::Captures, caps: &mut Self::Captures,
mut matched: F, mut matched: F,
) -> Result<(), Self::Error> ) -> Result<(), Self::Error>
where F: FnMut(&Self::Captures) -> bool where
F: FnMut(&Self::Captures) -> bool,
{ {
self.try_captures_iter(haystack, caps, |caps| Ok(matched(caps))) self.try_captures_iter(haystack, caps, |caps| Ok(matched(caps)))
.map(|r: Result<(), ()>| r.unwrap()) .map(|r: Result<(), ()>| r.unwrap())
@@ -709,7 +718,8 @@ pub trait Matcher {
caps: &mut Self::Captures, caps: &mut Self::Captures,
mut matched: F, mut matched: F,
) -> Result<Result<(), E>, Self::Error> ) -> Result<Result<(), E>, Self::Error>
where F: FnMut(&Self::Captures) -> Result<bool, E> where
F: FnMut(&Self::Captures) -> Result<bool, E>,
{ {
let mut last_end = 0; let mut last_end = 0;
let mut last_match = None; let mut last_match = None;
@@ -787,7 +797,8 @@ pub trait Matcher {
dst: &mut Vec<u8>, dst: &mut Vec<u8>,
mut append: F, mut append: F,
) -> Result<(), Self::Error> ) -> Result<(), Self::Error>
where F: FnMut(Match, &mut Vec<u8>) -> bool where
F: FnMut(Match, &mut Vec<u8>) -> bool,
{ {
let mut last_match = 0; let mut last_match = 0;
self.find_iter(haystack, |m| { self.find_iter(haystack, |m| {
@@ -810,7 +821,8 @@ pub trait Matcher {
dst: &mut Vec<u8>, dst: &mut Vec<u8>,
mut append: F, mut append: F,
) -> Result<(), Self::Error> ) -> Result<(), Self::Error>
where F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool where
F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool,
{ {
let mut last_match = 0; let mut last_match = 0;
self.captures_iter(haystack, caps, |caps| { self.captures_iter(haystack, caps, |caps| {
@@ -1012,10 +1024,7 @@ impl<'a, M: Matcher> Matcher for &'a M {
(*self).capture_count() (*self).capture_count()
} }
fn find( fn find(&self, haystack: &[u8]) -> Result<Option<Match>, Self::Error> {
&self,
haystack: &[u8]
) -> Result<Option<Match>, Self::Error> {
(*self).find(haystack) (*self).find(haystack)
} }
@@ -1024,7 +1033,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
haystack: &[u8], haystack: &[u8],
matched: F, matched: F,
) -> Result<(), Self::Error> ) -> Result<(), Self::Error>
where F: FnMut(Match) -> bool where
F: FnMut(Match) -> bool,
{ {
(*self).find_iter(haystack, matched) (*self).find_iter(haystack, matched)
} }
@@ -1034,7 +1044,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
haystack: &[u8], haystack: &[u8],
matched: F, matched: F,
) -> Result<Result<(), E>, Self::Error> ) -> Result<Result<(), E>, Self::Error>
where F: FnMut(Match) -> Result<bool, E> where
F: FnMut(Match) -> Result<bool, E>,
{ {
(*self).try_find_iter(haystack, matched) (*self).try_find_iter(haystack, matched)
} }
@@ -1053,7 +1064,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
caps: &mut Self::Captures, caps: &mut Self::Captures,
matched: F, matched: F,
) -> Result<(), Self::Error> ) -> Result<(), Self::Error>
where F: FnMut(&Self::Captures) -> bool where
F: FnMut(&Self::Captures) -> bool,
{ {
(*self).captures_iter(haystack, caps, matched) (*self).captures_iter(haystack, caps, matched)
} }
@@ -1064,7 +1076,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
caps: &mut Self::Captures, caps: &mut Self::Captures,
matched: F, matched: F,
) -> Result<Result<(), E>, Self::Error> ) -> Result<Result<(), E>, Self::Error>
where F: FnMut(&Self::Captures) -> Result<bool, E> where
F: FnMut(&Self::Captures) -> Result<bool, E>,
{ {
(*self).try_captures_iter(haystack, caps, matched) (*self).try_captures_iter(haystack, caps, matched)
} }
@@ -1075,7 +1088,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
dst: &mut Vec<u8>, dst: &mut Vec<u8>,
append: F, append: F,
) -> Result<(), Self::Error> ) -> Result<(), Self::Error>
where F: FnMut(Match, &mut Vec<u8>) -> bool where
F: FnMut(Match, &mut Vec<u8>) -> bool,
{ {
(*self).replace(haystack, dst, append) (*self).replace(haystack, dst, append)
} }
@@ -1087,7 +1101,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
dst: &mut Vec<u8>, dst: &mut Vec<u8>,
append: F, append: F,
) -> Result<(), Self::Error> ) -> Result<(), Self::Error>
where F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool where
F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool,
{ {
(*self).replace_with_captures(haystack, caps, dst, append) (*self).replace_with_captures(haystack, caps, dst, append)
} }
@@ -1099,7 +1114,7 @@ impl<'a, M: Matcher> Matcher for &'a M {
fn is_match_at( fn is_match_at(
&self, &self,
haystack: &[u8], haystack: &[u8],
at: usize at: usize,
) -> Result<bool, Self::Error> { ) -> Result<bool, Self::Error> {
(*self).is_match_at(haystack, at) (*self).is_match_at(haystack, at)
} }

172
grep-matcher/tests/test_matcher.rs → crates/matcher/tests/test_matcher.rs
View File

@@ -25,18 +25,22 @@ fn find() {
fn find_iter() { fn find_iter() {
let matcher = matcher(r"(\w+)\s+(\w+)"); let matcher = matcher(r"(\w+)\s+(\w+)");
let mut matches = vec![]; let mut matches = vec![];
matcher.find_iter(b"aa bb cc dd", |m| { matcher
matches.push(m); .find_iter(b"aa bb cc dd", |m| {
true matches.push(m);
}).unwrap(); true
})
.unwrap();
assert_eq!(matches, vec![m(0, 5), m(6, 11)]); assert_eq!(matches, vec![m(0, 5), m(6, 11)]);
// Test that find_iter respects short circuiting. // Test that find_iter respects short circuiting.
matches.clear(); matches.clear();
matcher.find_iter(b"aa bb cc dd", |m| { matcher
matches.push(m); .find_iter(b"aa bb cc dd", |m| {
false matches.push(m);
}).unwrap(); false
})
.unwrap();
assert_eq!(matches, vec![m(0, 5)]); assert_eq!(matches, vec![m(0, 5)]);
} }
@@ -47,14 +51,17 @@ fn try_find_iter() {
let matcher = matcher(r"(\w+)\s+(\w+)"); let matcher = matcher(r"(\w+)\s+(\w+)");
let mut matches = vec![]; let mut matches = vec![];
let err = matcher.try_find_iter(b"aa bb cc dd", |m| { let err = matcher
if matches.is_empty() { .try_find_iter(b"aa bb cc dd", |m| {
matches.push(m); if matches.is_empty() {
Ok(true) matches.push(m);
} else { Ok(true)
Err(MyError) } else {
} Err(MyError)
}).unwrap().unwrap_err(); }
})
.unwrap()
.unwrap_err();
assert_eq!(matches, vec![m(0, 5)]); assert_eq!(matches, vec![m(0, 5)]);
assert_eq!(err, MyError); assert_eq!(err, MyError);
} }
@@ -89,28 +96,30 @@ fn captures_iter() {
let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)"); let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)");
let mut caps = matcher.new_captures().unwrap(); let mut caps = matcher.new_captures().unwrap();
let mut matches = vec![]; let mut matches = vec![];
matcher.captures_iter(b"aa bb cc dd", &mut caps, |caps| { matcher
matches.push(caps.get(0).unwrap()); .captures_iter(b"aa bb cc dd", &mut caps, |caps| {
matches.push(caps.get(1).unwrap()); matches.push(caps.get(0).unwrap());
matches.push(caps.get(2).unwrap()); matches.push(caps.get(1).unwrap());
true matches.push(caps.get(2).unwrap());
}).unwrap(); true
assert_eq!(matches, vec![ })
m(0, 5), m(0, 2), m(3, 5), .unwrap();
m(6, 11), m(6, 8), m(9, 11), assert_eq!(
]); matches,
vec![m(0, 5), m(0, 2), m(3, 5), m(6, 11), m(6, 8), m(9, 11),]
);
// Test that captures_iter respects short circuiting. // Test that captures_iter respects short circuiting.
matches.clear(); matches.clear();
matcher.captures_iter(b"aa bb cc dd", &mut caps, |caps| { matcher
matches.push(caps.get(0).unwrap()); .captures_iter(b"aa bb cc dd", &mut caps, |caps| {
matches.push(caps.get(1).unwrap()); matches.push(caps.get(0).unwrap());
matches.push(caps.get(2).unwrap()); matches.push(caps.get(1).unwrap());
false matches.push(caps.get(2).unwrap());
}).unwrap(); false
assert_eq!(matches, vec![ })
m(0, 5), m(0, 2), m(3, 5), .unwrap();
]); assert_eq!(matches, vec![m(0, 5), m(0, 2), m(3, 5),]);
} }
#[test] #[test]
@@ -121,16 +130,19 @@ fn try_captures_iter() {
let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)"); let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)");
let mut caps = matcher.new_captures().unwrap(); let mut caps = matcher.new_captures().unwrap();
let mut matches = vec![]; let mut matches = vec![];
let err = matcher.try_captures_iter(b"aa bb cc dd", &mut caps, |caps| { let err = matcher
if matches.is_empty() { .try_captures_iter(b"aa bb cc dd", &mut caps, |caps| {
matches.push(caps.get(0).unwrap()); if matches.is_empty() {
matches.push(caps.get(1).unwrap()); matches.push(caps.get(0).unwrap());
matches.push(caps.get(2).unwrap()); matches.push(caps.get(1).unwrap());
Ok(true) matches.push(caps.get(2).unwrap());
} else { Ok(true)
Err(MyError) } else {
} Err(MyError)
}).unwrap().unwrap_err(); }
})
.unwrap()
.unwrap_err();
assert_eq!(matches, vec![m(0, 5), m(0, 2), m(3, 5)]); assert_eq!(matches, vec![m(0, 5), m(0, 2), m(3, 5)]);
assert_eq!(err, MyError); assert_eq!(err, MyError);
} }
@@ -150,10 +162,12 @@ fn no_captures() {
assert!(!matcher.captures(b"homer simpson", &mut caps).unwrap()); assert!(!matcher.captures(b"homer simpson", &mut caps).unwrap());
let mut called = false; let mut called = false;
matcher.captures_iter(b"homer simpson", &mut caps, |_| { matcher
called = true; .captures_iter(b"homer simpson", &mut caps, |_| {
true called = true;
}).unwrap(); true
})
.unwrap();
assert!(!called); assert!(!called);
} }
@@ -161,18 +175,22 @@ fn no_captures() {
fn replace() { fn replace() {
let matcher = matcher(r"(\w+)\s+(\w+)"); let matcher = matcher(r"(\w+)\s+(\w+)");
let mut dst = vec![]; let mut dst = vec![];
matcher.replace(b"aa bb cc dd", &mut dst, |_, dst| { matcher
dst.push(b'z'); .replace(b"aa bb cc dd", &mut dst, |_, dst| {
true dst.push(b'z');
}).unwrap(); true
})
.unwrap();
assert_eq!(dst, b"z z"); assert_eq!(dst, b"z z");
// Test that replacements respect short circuiting. // Test that replacements respect short circuiting.
dst.clear(); dst.clear();
matcher.replace(b"aa bb cc dd", &mut dst, |_, dst| { matcher
dst.push(b'z'); .replace(b"aa bb cc dd", &mut dst, |_, dst| {
false dst.push(b'z');
}).unwrap(); false
})
.unwrap();
assert_eq!(dst, b"z cc dd"); assert_eq!(dst, b"z cc dd");
} }
@@ -182,27 +200,31 @@ fn replace_with_captures() {
let haystack = b"aa bb cc dd"; let haystack = b"aa bb cc dd";
let mut caps = matcher.new_captures().unwrap(); let mut caps = matcher.new_captures().unwrap();
let mut dst = vec![]; let mut dst = vec![];
matcher.replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| { matcher
caps.interpolate( .replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| {
|name| matcher.capture_index(name), caps.interpolate(
haystack, |name| matcher.capture_index(name),
b"$2 $1", haystack,
dst, b"$2 $1",
); dst,
true );
}).unwrap(); true
})
.unwrap();
assert_eq!(dst, b"bb aa dd cc"); assert_eq!(dst, b"bb aa dd cc");
// Test that replacements respect short circuiting. // Test that replacements respect short circuiting.
dst.clear(); dst.clear();
matcher.replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| { matcher
caps.interpolate( .replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| {
|name| matcher.capture_index(name), caps.interpolate(
haystack, |name| matcher.capture_index(name),
b"$2 $1", haystack,
dst, b"$2 $1",
); dst,
false );
}).unwrap(); false
})
.unwrap();
assert_eq!(dst, b"bb aa cc dd"); assert_eq!(dst, b"bb aa cc dd");
} }

0
grep-matcher/tests/tests.rs → crates/matcher/tests/tests.rs
View File

23
grep-matcher/tests/util.rs → crates/matcher/tests/util.rs
View File

@@ -18,10 +18,7 @@ impl RegexMatcher {
names.insert(name.to_string(), i); names.insert(name.to_string(), i);
} }
} }
RegexMatcher { RegexMatcher { re: re, names: names }
re: re,
names: names,
}
} }
} }
@@ -31,12 +28,9 @@ impl Matcher for RegexMatcher {
type Captures = RegexCaptures; type Captures = RegexCaptures;
type Error = NoError; type Error = NoError;
fn find_at( fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>> {
&self, Ok(self
haystack: &[u8], .re
at: usize,
) -> Result<Option<Match>> {
Ok(self.re
.find_at(haystack, at) .find_at(haystack, at)
.map(|m| Match::new(m.start(), m.end()))) .map(|m| Match::new(m.start(), m.end())))
} }
@@ -75,12 +69,9 @@ impl Matcher for RegexMatcherNoCaps {
type Captures = NoCaptures; type Captures = NoCaptures;
type Error = NoError; type Error = NoError;
fn find_at( fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>> {
&self, Ok(self
haystack: &[u8], .0
at: usize,
) -> Result<Option<Match>> {
Ok(self.0
.find_at(haystack, at) .find_at(haystack, at)
.map(|m| Match::new(m.start(), m.end()))) .map(|m| Match::new(m.start(), m.end())))
} }

8
grep-pcre2/Cargo.toml → crates/pcre2/Cargo.toml
View File

@@ -1,17 +1,17 @@
[package] [package]
name = "grep-pcre2" name = "grep-pcre2"
version = "0.1.3" #:version version = "0.1.4" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"] authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """ description = """
Use PCRE2 with the 'grep' crate. Use PCRE2 with the 'grep' crate.
""" """
documentation = "https://docs.rs/grep-pcre2" documentation = "https://docs.rs/grep-pcre2"
homepage = "https://github.com/BurntSushi/ripgrep" homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/pcre2"
repository = "https://github.com/BurntSushi/ripgrep" repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/pcre2"
readme = "README.md" readme = "README.md"
keywords = ["regex", "grep", "pcre", "backreference", "look"] keywords = ["regex", "grep", "pcre", "backreference", "look"]
license = "Unlicense/MIT" license = "Unlicense/MIT"
[dependencies] [dependencies]
grep-matcher = { version = "0.1.2", path = "../grep-matcher" } grep-matcher = { version = "0.1.2", path = "../matcher" }
pcre2 = "0.2.0" pcre2 = "0.2.0"

0
grep-regex/LICENSE-MIT → crates/pcre2/LICENSE-MIT
View File

0
grep-pcre2/README.md → crates/pcre2/README.md
View File

0
grep-regex/UNLICENSE → crates/pcre2/UNLICENSE
View File

0
grep-pcre2/src/error.rs → crates/pcre2/src/error.rs
View File

Some files were not shown because too many files have changed in this diff Show More