twin/ripgrep
1
0
Fork 0
mirror of https://github.com/BurntSushi/ripgrep.git synced 2025-07-26 17:51:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: twin:grep-matcher-0.1.1
Branches Tags
twin:master twin:ag/fix-ci twin:ag/index twin:ag/fix-multiline-replace-duplicate twin:ag/fix-cve-2021-3013 twin:ag/moarci twin:ag/ghactions twin:ag/snafu twin:ag/disable-bom-sniff twin:ag/partial-bstr-migration twin:ag/bstr-migration twin:ag/prepare-0.10.0 twin:ag/libripgrep-freeze-2 twin:ag/libripgrep-freeze-1 twin:ag/changelog
twin:globset-0.4.16 twin:14.1.1 twin:14.1.0 twin:ignore-0.4.22 twin:14.0.3 twin:14.0.2 twin:grep-0.3.1 twin:grep-printer-0.2.1 twin:grep-searcher-0.1.13 twin:14.0.1 twin:grep-0.3.0 twin:grep-printer-0.2.0 twin:14.0.0 twin:grep-0.2.13 twin:grep-searcher-0.1.12 twin:grep-pcre2-0.1.7 twin:grep-regex-0.1.12 twin:grep-matcher-0.1.7 twin:grep-cli-0.1.10 twin:ignore-0.4.21 twin:globset-0.4.14 twin:globset-0.4.13 twin:globset-0.4.12 twin:grep-cli-0.1.9 twin:globset-0.4.11 twin:grep-cli-0.1.8 twin:grep-0.2.12 twin:ignore-0.4.20 twin:grep-0.2.11 twin:grep-printer-0.1.7 twin:grep-searcher-0.1.11 twin:grep-pcre2-0.1.6 twin:grep-regex-0.1.11 twin:grep-matcher-0.1.6 twin:grep-cli-0.1.7 twin:ignore-0.4.19 twin:globset-0.4.10 twin:grep-0.2.10 twin:grep-searcher-0.1.10 twin:grep-0.2.9 twin:grep-searcher-0.1.9 twin:grep-regex-0.1.10 twin:globset-0.4.9 twin:globset-0.4.8 twin:13.0.0 twin:grep-0.2.8 twin:grep-printer-0.1.6 twin:grep-searcher-0.1.8 twin:grep-pcre2-0.1.5 twin:grep-regex-0.1.9 twin:grep-matcher-0.1.5 twin:grep-cli-0.1.6 twin:ignore-0.4.18 twin:globset-0.4.7 twin:ignore-0.4.17 twin:globset-0.4.6 twin:12.1.1 twin:grep-0.2.7 twin:grep-cli-0.1.5 twin:ignore-0.4.16 twin:12.1.0 twin:grep-0.2.6 twin:grep-printer-0.1.5 twin:grep-regex-0.1.8 twin:ignore-0.4.15 twin:12.0.1 twin:ignore-0.4.14 twin:ignore-0.4.13 twin:grep-regex-0.1.7 twin:12.0.0 twin:grep-searcher-0.1.7 twin:grep-regex-0.1.6 twin:grep-printer-0.1.4 twin:grep-pcre2-0.1.4 twin:grep-matcher-0.1.4 twin:ignore-0.4.12 twin:grep-0.2.5 twin:globset-0.4.5 twin:grep-cli-0.1.4 twin:ignore-0.4.11 twin:grep-regex-0.1.5 twin:grep-searcher-0.1.6 twin:ignore-0.4.10 twin:11.0.2 twin:grep-matcher-0.1.3 twin:grep-regex-0.1.4 twin:ignore-0.4.9 twin:ignore-0.4.8 twin:grep-searcher-0.1.5 twin:grep-printer-0.1.3 twin:grep-cli-0.1.3 twin:globset-0.4.4 twin:11.0.1 twin:11.0.0 twin:grep-0.2.4 twin:grep-cli-0.1.2 twin:grep-printer-0.1.2 twin:grep-searcher-0.1.4 twin:grep-regex-0.1.3 twin:grep-pcre2-0.1.3 twin:grep-matcher-0.1.2 twin:ignore-0.4.7 twin:globset-0.4.3 twin:grep-regex-0.1.2 twin:grep-searcher-0.1.3 twin:grep-searcher-0.1.2 twin:ignore-0.4.6 twin:ignore-0.4.5 twin:grep-0.2.3 twin:grep-pcre2-0.1.2 twin:0.10.0 twin:grep-searcher-0.1.1 twin:globset-0.4.2 twin:grep-pcre2-0.1.1 twin:grep-0.2.2 twin:grep-printer-0.1.1 twin:ignore-0.4.4 twin:grep-regex-0.1.1 twin:grep-matcher-0.1.1 twin:grep-cli-0.1.1 twin:grep-0.2.1 twin:grep-cli-0.1.0 twin:grep-0.2.0 twin:grep-search-0.1.0 twin:grep-matcher-0.1.0 twin:grep-printer-0.1.0 twin:grep-pcre2-0.1.0 twin:grep-regex-0.1.0 twin:0.9.0 twin:grep-0.1.9 twin:ignore-0.4.3 twin:globset-0.4.1 twin:ignore-0.4.2 twin:globset-0.4.0 twin:termcolor-0.3.6 twin:0.8.1 twin:termcolor-0.3.5 twin:ignore-0.4.1 twin:0.8.0 twin:ignore-0.4.0 twin:globset-0.3.0 twin:termcolor-0.3.4 twin:wincolor-0.1.6 twin:grep-0.1.8 twin:wincolor-0.1.5 twin:0.7.1 twin:ignore-0.3.1 twin:ignore-0.3.0 twin:grep-0.1.7 twin:globset-0.2.1 twin:0.7.0 twin:termcolor-0.3.3 twin:0.6.0 twin:ignore-0.2.2 twin:ignore-0.2.1 twin:wincolor-0.1.4 twin:0.5.2 twin:ignore-0.1.9 twin:0.5.1 twin:termcolor-0.3.2 twin:0.5.0 twin:grep-0.1.6 twin:ignore-0.1.8 twin:globset-0.1.4 twin:termcolor-0.3.1 twin:wincolor-0.1.3 twin:termcolor-0.3.0 twin:wincolor-0.1.2 twin:0.4.0 twin:0.3.2 twin:ignore-0.1.6 twin:0.3.1 twin:termcolor-0.1.1 twin:0.3.0 twin:termcolor-0.1.0 twin:wincolor-0.1.0 twin:0.2.9 twin:ignore-0.1.5 twin:0.2.8 twin:0.2.7 twin:ignore-0.1.4 twin:grep-0.1.4 twin:globset-0.1.2 twin:0.2.6 twin:ignore-0.1.3 twin:ignore-0.1.2 twin:0.2.5 twin:0.2.4 twin:ignore-0.1.1 twin:ignore-0.1.0 twin:globset-0.1.1 twin:0.2.3 twin:0.2.2 twin:globset-0.1.0 twin:0.2.1 twin:0.2.0 twin:0.1.18 twin:0.1.17 twin:0.1.16 twin:0.1.15 twin:0.1.14 twin:0.1.13 twin:0.1.12 twin:0.1.11 twin:0.1.10 twin:0.1.9 twin:0.1.8 twin:0.1.7 twin:0.1.6 twin:0.1.5 twin:0.1.4 twin:0.1.3 twin:0.1.2 twin:0.1.1 twin:0.1.0 twin:0.0.19 twin:0.0.18 twin:0.0.17 twin:0.0.16 twin:0.0.15 twin:0.0.14 twin:0.0.13 twin:0.0.12 twin:0.0.11 twin:0.0.10 twin:0.0.9 twin:0.0.8 twin:0.0.7 twin:0.0.6 twin:0.0.5 twin:0.0.4 twin:0.0.3 twin:0.0.2 twin:0.0.1
...
compare: twin:12.1.1
Branches Tags
twin:master twin:ag/fix-ci twin:ag/index twin:ag/fix-multiline-replace-duplicate twin:ag/fix-cve-2021-3013 twin:ag/moarci twin:ag/ghactions twin:ag/snafu twin:ag/disable-bom-sniff twin:ag/partial-bstr-migration twin:ag/bstr-migration twin:ag/prepare-0.10.0 twin:ag/libripgrep-freeze-2 twin:ag/libripgrep-freeze-1 twin:ag/changelog
twin:globset-0.4.16 twin:14.1.1 twin:14.1.0 twin:ignore-0.4.22 twin:14.0.3 twin:14.0.2 twin:grep-0.3.1 twin:grep-printer-0.2.1 twin:grep-searcher-0.1.13 twin:14.0.1 twin:grep-0.3.0 twin:grep-printer-0.2.0 twin:14.0.0 twin:grep-0.2.13 twin:grep-searcher-0.1.12 twin:grep-pcre2-0.1.7 twin:grep-regex-0.1.12 twin:grep-matcher-0.1.7 twin:grep-cli-0.1.10 twin:ignore-0.4.21 twin:globset-0.4.14 twin:globset-0.4.13 twin:globset-0.4.12 twin:grep-cli-0.1.9 twin:globset-0.4.11 twin:grep-cli-0.1.8 twin:grep-0.2.12 twin:ignore-0.4.20 twin:grep-0.2.11 twin:grep-printer-0.1.7 twin:grep-searcher-0.1.11 twin:grep-pcre2-0.1.6 twin:grep-regex-0.1.11 twin:grep-matcher-0.1.6 twin:grep-cli-0.1.7 twin:ignore-0.4.19 twin:globset-0.4.10 twin:grep-0.2.10 twin:grep-searcher-0.1.10 twin:grep-0.2.9 twin:grep-searcher-0.1.9 twin:grep-regex-0.1.10 twin:globset-0.4.9 twin:globset-0.4.8 twin:13.0.0 twin:grep-0.2.8 twin:grep-printer-0.1.6 twin:grep-searcher-0.1.8 twin:grep-pcre2-0.1.5 twin:grep-regex-0.1.9 twin:grep-matcher-0.1.5 twin:grep-cli-0.1.6 twin:ignore-0.4.18 twin:globset-0.4.7 twin:ignore-0.4.17 twin:globset-0.4.6 twin:12.1.1 twin:grep-0.2.7 twin:grep-cli-0.1.5 twin:ignore-0.4.16 twin:12.1.0 twin:grep-0.2.6 twin:grep-printer-0.1.5 twin:grep-regex-0.1.8 twin:ignore-0.4.15 twin:12.0.1 twin:ignore-0.4.14 twin:ignore-0.4.13 twin:grep-regex-0.1.7 twin:12.0.0 twin:grep-searcher-0.1.7 twin:grep-regex-0.1.6 twin:grep-printer-0.1.4 twin:grep-pcre2-0.1.4 twin:grep-matcher-0.1.4 twin:ignore-0.4.12 twin:grep-0.2.5 twin:globset-0.4.5 twin:grep-cli-0.1.4 twin:ignore-0.4.11 twin:grep-regex-0.1.5 twin:grep-searcher-0.1.6 twin:ignore-0.4.10 twin:11.0.2 twin:grep-matcher-0.1.3 twin:grep-regex-0.1.4 twin:ignore-0.4.9 twin:ignore-0.4.8 twin:grep-searcher-0.1.5 twin:grep-printer-0.1.3 twin:grep-cli-0.1.3 twin:globset-0.4.4 twin:11.0.1 twin:11.0.0 twin:grep-0.2.4 twin:grep-cli-0.1.2 twin:grep-printer-0.1.2 twin:grep-searcher-0.1.4 twin:grep-regex-0.1.3 twin:grep-pcre2-0.1.3 twin:grep-matcher-0.1.2 twin:ignore-0.4.7 twin:globset-0.4.3 twin:grep-regex-0.1.2 twin:grep-searcher-0.1.3 twin:grep-searcher-0.1.2 twin:ignore-0.4.6 twin:ignore-0.4.5 twin:grep-0.2.3 twin:grep-pcre2-0.1.2 twin:0.10.0 twin:grep-searcher-0.1.1 twin:globset-0.4.2 twin:grep-pcre2-0.1.1 twin:grep-0.2.2 twin:grep-printer-0.1.1 twin:ignore-0.4.4 twin:grep-regex-0.1.1 twin:grep-matcher-0.1.1 twin:grep-cli-0.1.1 twin:grep-0.2.1 twin:grep-cli-0.1.0 twin:grep-0.2.0 twin:grep-search-0.1.0 twin:grep-matcher-0.1.0 twin:grep-printer-0.1.0 twin:grep-pcre2-0.1.0 twin:grep-regex-0.1.0 twin:0.9.0 twin:grep-0.1.9 twin:ignore-0.4.3 twin:globset-0.4.1 twin:ignore-0.4.2 twin:globset-0.4.0 twin:termcolor-0.3.6 twin:0.8.1 twin:termcolor-0.3.5 twin:ignore-0.4.1 twin:0.8.0 twin:ignore-0.4.0 twin:globset-0.3.0 twin:termcolor-0.3.4 twin:wincolor-0.1.6 twin:grep-0.1.8 twin:wincolor-0.1.5 twin:0.7.1 twin:ignore-0.3.1 twin:ignore-0.3.0 twin:grep-0.1.7 twin:globset-0.2.1 twin:0.7.0 twin:termcolor-0.3.3 twin:0.6.0 twin:ignore-0.2.2 twin:ignore-0.2.1 twin:wincolor-0.1.4 twin:0.5.2 twin:ignore-0.1.9 twin:0.5.1 twin:termcolor-0.3.2 twin:0.5.0 twin:grep-0.1.6 twin:ignore-0.1.8 twin:globset-0.1.4 twin:termcolor-0.3.1 twin:wincolor-0.1.3 twin:termcolor-0.3.0 twin:wincolor-0.1.2 twin:0.4.0 twin:0.3.2 twin:ignore-0.1.6 twin:0.3.1 twin:termcolor-0.1.1 twin:0.3.0 twin:termcolor-0.1.0 twin:wincolor-0.1.0 twin:0.2.9 twin:ignore-0.1.5 twin:0.2.8 twin:0.2.7 twin:ignore-0.1.4 twin:grep-0.1.4 twin:globset-0.1.2 twin:0.2.6 twin:ignore-0.1.3 twin:ignore-0.1.2 twin:0.2.5 twin:0.2.4 twin:ignore-0.1.1 twin:ignore-0.1.0 twin:globset-0.1.1 twin:0.2.3 twin:0.2.2 twin:globset-0.1.0 twin:0.2.1 twin:0.2.0 twin:0.1.18 twin:0.1.17 twin:0.1.16 twin:0.1.15 twin:0.1.14 twin:0.1.13 twin:0.1.12 twin:0.1.11 twin:0.1.10 twin:0.1.9 twin:0.1.8 twin:0.1.7 twin:0.1.6 twin:0.1.5 twin:0.1.4 twin:0.1.3 twin:0.1.2 twin:0.1.1 twin:0.1.0 twin:0.0.19 twin:0.0.18 twin:0.0.17 twin:0.0.16 twin:0.0.15 twin:0.0.14 twin:0.0.13 twin:0.0.12 twin:0.0.11 twin:0.0.10 twin:0.0.9 twin:0.0.8 twin:0.0.7 twin:0.0.6 twin:0.0.5 twin:0.0.4 twin:0.0.3 twin:0.0.2 twin:0.0.1

382 Commits
grep-match ... 12.1.1

Author SHA1 Message Date
Andrew Gallant
7cb211378a 12.1.1 2020-05-29 09:26:47 -04:00
Andrew Gallant
a73c0a21d9 changelog: 12.1.1 2020-05-29 09:26:33 -04:00
Andrew Gallant
0b965f900c doc: small release checklist updates 
In particular, explicitly note when to update the CHANGELOG.

Also, tweak the ripgrep introductory message.
 2020-05-29 09:21:19 -04:00
Andrew Gallant
a2f90747c9 core: update minimal dependency versions 2020-05-29 09:18:59 -04:00
Andrew Gallant
f97cc623f7 grep-0.2.7 2020-05-29 09:17:24 -04:00
Andrew Gallant
f35de5c523 grep: update minimal dependency versions 2020-05-29 09:17:08 -04:00
Andrew Gallant
c9bb78ceba grep-cli-0.1.5 2020-05-29 09:14:18 -04:00
Andrew Gallant
72bdde6771 ignore-0.4.16 2020-05-29 09:13:02 -04:00
Andrew Gallant
d66712a452 deps: update all dependencies 2020-05-29 09:11:50 -04:00
Andy Salerno
e8822ce97a ignore/doc: update misleading documentation 
This likely originated from a bad copy/paste.

PR #1596
 2020-05-24 23:12:53 -04:00
Andrew Gallant
a700b75843 doc: clarify capture group indices 
And in particular, note the special $0 index, which corresponds to the
entire match.

Fixes #1591
 2020-05-21 22:22:51 -04:00
Gerion Entrup
b72ad8f8aa ignore/types: add meson filetype 
Closes #1586, PR #1587
 2020-05-18 14:01:35 -04:00
Andrew Gallant
1980630f17 doc: fix egregious markup output 
We use '+++' syntax to output a literal '**' for a '--glob' example.
This '+++' syntax is pretty ugly when rendered literally via --help. We
fix this by hackily inserting the '+++' syntax for its one specific case
that we need it during man page generation.

Not ideal but it works. And --help still has some '*foo*' markup, but we
live with that for now.

Fixes #1581
 2020-05-13 08:13:05 -04:00
Andrew Gallant
1e9a481a66 doc: more release checklist updates 2020-05-09 11:43:37 -04:00
Andrew Gallant
bacfca174e pkg: update brew tap to version 12.1.0 
This also removes Fish shell completions. See #1577 for more details.
 2020-05-09 11:42:39 -04:00
Andrew Gallant
6162b000a3 changelog: 12.1.0 2020-05-09 11:36:44 -04:00
Andrew Gallant
2658bd4e46 12.1.0 2020-05-09 11:13:33 -04:00
Andrew Gallant
4b8e1f030e doc: add more detail to release checklist 
Getting the crate order right is important, so document it.
 2020-05-09 11:12:51 -04:00
Andrew Gallant
72807462e8 deps: update minimal versions for dependencies 2020-05-09 10:39:43 -04:00
Andrew Gallant
08dee094dd grep-0.2.6 2020-05-09 10:37:29 -04:00
Andrew Gallant
caa53b7b09 grep: update minimal dependency versions 2020-05-09 10:37:08 -04:00
Andrew Gallant
c5d6141562 grep-printer-0.1.5 2020-05-09 10:33:02 -04:00
Andrew Gallant
c0f0492b98 grep-regex-0.1.8 2020-05-09 10:31:29 -04:00
Andrew Gallant
568018386b ignore-0.4.15 2020-05-09 10:27:19 -04:00
Andrew Gallant
6219d29c24 doc: add 'cargo outdated' step to release checklist 
It's just good sense to make sure everything is updated if possible.
 2020-05-09 10:26:00 -04:00
Andrew Gallant
b458cf39f2 deps: update to base64 0.12 
No code changes were necessary.
 2020-05-09 10:25:37 -04:00
Andrew Gallant
3fd2694fbc deps: update all dependencies 
Everything looks pretty minor.
 2020-05-09 09:05:51 -04:00
Andrew Gallant
b56315ea84 changelog: add #1550 to CHANGELOG 2020-05-08 23:37:17 -04:00
Andrew Gallant
fac47906e6 doc: add a release checklist 
The steps are numerous, subtle and complex enough that it's worth
writing them down. In particular, getting the order correct is
important. (i.e., If we released to crates.io first and the GitHub
release infrastructure failed, then we'd be in a pickle.)
 2020-05-08 23:24:40 -04:00
Andrew Gallant
e02bb6b99a changelog: add downstream notices 2020-05-08 23:24:40 -04:00
Chayoung You
16a1221fc7 doc: use asciidoctor instead of a2x 
AsciiDoc development is continued under asciidoctor. See
https://github.com/asciidoc/asciidoc.

We do however fallback to a2x if asciidoctor is not present. This is to
ease migration, but at some point, it's likely that support for a2x will
be dropped.

Originally reported downstream:
https://github.com/Homebrew/linuxbrew-core/issues/19885

Closes #1544
 2020-05-08 23:24:40 -04:00
Casey Rodarmor
793c1179cc ignore: allow filtering with predicate 
Adds `WalkBuilder::filter_entry` that takes a predicate to be applied to
all entries. If the predicate returns `false` on a given entry, that
entry and all children will be skipped.

Fixes #1555, Closes #1557
 2020-05-08 23:24:40 -04:00
Wieland Hoffmann
df7a3bfc7f grep-cli: support files compressed by compress(1) 
While Linux distributions (at least Arch Linux, RHEL, Debian) do not support
compressing files with compress(1), macOS & AIX do (the utility is part of
POSIX). Additionally, gzip is able to uncompress such compressed files and
provides an `uncompress` binary.

Closes #1547
 2020-05-08 23:24:40 -04:00
Andrew Gallant
28f2a93cae doc: shorten -h/--help prelude 
It has grown quite long. It would be nice if we could shorten this only
when -h is used and keep it long for --help, but it seems clap doesn't
let this happen. (It does have `about` and `long_about` options, but
they don't work, even when I disable the use of the template.)

The longer prelude is now only available in the man page.

This addresses #189.
 2020-05-08 23:24:40 -04:00
Andrew Gallant
0eb2501b6e doc: add a section about --pre to the GUIDE 
Fixes #1252
 2020-05-08 23:24:40 -04:00
Andrew Gallant
184c15882e doc: add -U/--multiline to common options 2020-05-08 23:24:40 -04:00
Andrew Gallant
64a4dee495 cli: improve invalid UTF-8 pattern error message 
When a pattern with invalid UTF-8 is given, the error message suggests
unqualified use of hex escape sequences to match arbitrary bytes. But
you *also* need to disable Unicode mode. So include that in the error
message.

Fixes #1339
 2020-05-08 23:24:40 -04:00
Andrew Gallant
50840ea43b doc: note how to escape a '$' in --replace 
Fixes #1524
 2020-05-08 23:24:40 -04:00
Andrew Gallant
17dcc2bf51 doc: clarify that *files* override gitignores 
This attempts to fix some mild confusion that came up as part of #1574.
Specifically:
https://github.com/BurntSushi/ripgrep/issues/1574#issuecomment-625780436
 2020-05-08 23:24:40 -04:00
Andrew Gallant
9a858e4909 doc: add config file note for --type-{add,clear} 
This clarifies that persistence is possible via a configuration file.

Fixes #1571
 2020-05-08 23:24:40 -04:00
Andrew Gallant
cbfbe9312f snap: remove snapcraft configuration 
This hasn't been updated in ages and it's not clear what purpose it's
serving.
 2020-05-08 23:24:40 -04:00
Andrew Gallant
7ed9a31819 printer: fix --count-matches output 
In order to implement --count-matches, we simply re-execute the regex on
the spans reported by the searcher. The spans always correspond to the
lines that participated in the match. This is the correct thing to do,
except when the regex contains look-ahead (or look-behind).

In particular, the look-around permits the regex's match success to
depends on an arbitrary point before or after the lines actually
reported as participating in the match. Since only the matched lines are
reported to the printer, it is possible for subsequent searching on
those lines to fail.

A true fix for this would somehow make the total span available to the
printer. But that seems tricky since it isn't always available. For
PCRE2's case in multiline mode, it is available because we force it to
be so for correctness.

For now, we simply detect this corner case heuristically. If the match
count is zero, then it necessarily means there is some kind of
look-around that isn't matching. So we set the match count to 1. This is
probably incorrect in some cases, although my brain can't quite come up
with a concrete example. Nevertheless, this is strictly better than the
status quo.

Fixes #1573
 2020-05-08 23:24:40 -04:00
Andrew Gallant
a2e6aec7a4 tests: add new regression test for fixed inner literal bug 
This adds a new test case for a bug (#1537) that has already been fixed.
Or more precisely, a new bug with the same root cause.

Closes #1559
 2020-04-23 08:37:04 -04:00
Andrew Gallant
73103df6d9 deps: small dependency updates 2020-04-18 11:33:27 -04:00
Andrew Gallant
139f186e57 crates/ignore: switch to depth first traversal 
This replaces the use of channels in the parallel directory traversal
with a simple stack. The primary motivation for this change is to reduce
peak memory usage. In particular, when using a channel (which is a
queue), we wind up visiting files in a breadth first fashion. Using a
stack switches us to a depth first traversal. While there are no real
intrinsic differences, depth first traversal generally tends to use less
memory because directory trees are more commonly wide than they are
deep.

In particular, the queue/stack size itself is not the only concern. In
one recent case documented in #1550, a user wanted to search all Rust
crates. The directory structure was shallow but extremely wide, with a
single directory containing all crates. This in turn results is in
descending into each of those directories and building a gitignore
matcher for each (since most crates have `.gitignore` files) before ever
searching a single file. This means that ripgrep has all such matchers
in memory simultaneously, which winds up using quite a bit of memory.

In a depth first traversal, peak memory usage is much lower because
gitignore matches are built and discarded more quickly. In the case of
searching all crates, the peak memory usage decrease is dramatic. On my
system, it shrinks by an order magnitude, from almost 1GB to 50MB. The
decline in peak memory usage is consistent across other use cases as
well, but is typically more modest. For example, searching the Linux
repo has a 50% decrease in peak memory usage and searching the Chromium
repo has a 25% decrease in peak memory usage.

Search times generally remain unchanged, although some ad hoc benchmarks
that I typically run have gotten a bit slower. As far as I can tell,
this appears to be result of scheduling changes. Namely, the depth first
traversal seems to result in searching some very large files towards the
end of the search, which reduces the effectiveness of parallelism and
makes the overall search take longer. This seems to suggest that a stack
isn't optimal. It would instead perhaps be better to prioritize
searching larger files first, but it's not quite clear how to do this
without introducing more overhead (getting the file size for each file
requires a stat call).

Fixes #1550
 2020-04-18 11:33:03 -04:00
Andrew Gallant
afb325f733 readme: fix ordering of benchmarks 
Results remain the same. I just didn't order them correctly.
 2020-04-16 12:03:46 -04:00
Andrew Gallant
40af352d74 github: add necessary metadata 2020-04-14 16:28:09 -04:00
Andrew Gallant
3f1d4b397d github: switch to new issue template format 
And also point folks toward Discussions.
 2020-04-14 16:23:47 -04:00
Andrew Gallant
a75b4d122a doc: fix newline escape 
Fixes #1551
 2020-04-13 08:49:27 -04:00
Simon Robin
f51b762c6d pkg: fix brew tap version 
It wasn't updated after the 12.0.1 release, even though the
SHA values were.

PR #1545
 2020-04-07 19:45:53 -04:00
Andrew Gallant
49de7b119c ci: disable man page check 
It appears to be intermittently failing. Specifically, a2x seems to be
failing occasionally with no apparent reason why. The error message it
gives is inscrutable. Sigh.
 2020-04-01 21:18:04 -04:00
Andrew Gallant
1c4b5adb7b regex: fix another inner literal bug 
It looks like `is_simple` wasn't quite correct.

I can't wait until this code is rewritten. It is still not quite clearly
correct to me.

Fixes #1537
 2020-04-01 20:37:48 -04:00
Marius Schulz
3d6a58faff doc: fix typo in help description 
PR #1536
 2020-03-30 17:31:16 -04:00
Andrew Gallant
5b6ca04e39 ci: upgrade to actions/checkout@v2 
In particular, this appears to fix an extremely annoying bug that was
causing PR builds to fail if they were re-run.

For more details:
https://github.com/actions/checkout/issues/23#issuecomment-572688577
 2020-03-30 17:09:41 -04:00
Andrew Gallant
47f20c2661 pkg: update brew tap to 12.0.1 2020-03-29 19:18:57 -04:00
Andrew Gallant
1d5b1011e5 12.0.1 2020-03-29 18:59:40 -04:00
Andrew Gallant
1bb30b72fc changelog: prepare for 12.0.1 release, redux 2020-03-29 18:50:31 -04:00
Andrew Gallant
09a4b75baf ignore-0.4.14 2020-03-29 18:49:01 -04:00
Andrew Gallant
58c428827d changelog: prepare for 12.0.1 release 2020-03-29 18:47:46 -04:00
Andrew Gallant
b9bb04b793 deps: minor dependency updates 2020-03-29 18:47:15 -04:00
Zoltan Puskas
4dfea016b9 ignore/types: add ebuild type 
Add support for Gentoo's portage package manager spec files:
https://wiki.gentoo.org/wiki/Portage
 2020-03-29 18:44:04 -04:00
Andrew Gallant
3193d57ac1 ci: attempt to fix CI 
It looks like a2x isn't working, so take a shot at fixing it.
 2020-03-28 21:36:29 -04:00
Andrew Gallant
67c0f576b6 ignore-0.4.13 2020-03-22 21:08:37 -04:00
Andrew Gallant
543f99dbf1 grep-regex-0.1.7 2020-03-22 21:08:19 -04:00
Andrew Gallant
0ea65efd6d regex: special case literal extraction 
In a prior commit, we fixed a performance problem with the -w flag by
doing a little extra work to extract literals. It turns out that using
literals in this case when the -w flag is NOT used results in a
performance regression. The reasoning is that we end up using a "fast"
regex as a prefilter when the regex engine itself uses its own
equivalent prefilter, so ripgrep ends up redoing a fair amount of work.

Instead, we only do this extra work when we know the -w flag is enabled.
 2020-03-22 21:02:51 -04:00
Paul A. Patience
20deae6497 tests: fix typo in test name 
PR #1528
 2020-03-22 07:43:16 -04:00
Andrew Gallant
655e33219a crates.io: remove badges 
... and don't replace them with anything because crates.io does not
support GitHub Actions yet. But it's almost there:
https://github.com/rust-lang/crates.io/pull/1838

Thanks @atouchet for noticing this.
 2020-03-17 17:50:37 -04:00
Andrew Gallant
8ba6ccd159 ignore: fix failing test 
This fixes fallout from fixing #1520.
 2020-03-16 19:16:24 -04:00
Andrew Gallant
34edb8123a ignore: squash noisy error message 
We should not assume that the commondir file actually exists. If it
doesn't, then just move on. This otherwise emits an error message when
searching normal submodules, which is not OK.

This regression was introduced in #1446.

Fixes #1520
 2020-03-16 18:50:02 -04:00
Andrew Gallant
5b30c2aed6 ci: fix deb build script 2020-03-15 22:11:32 -04:00
Andrew Gallant
bf1027a83e pkg: update brew tap to 12.0.0 2020-03-15 22:10:08 -04:00
Andrew Gallant
031264e5fb ci: tweak release name 
This is consistent with prior releases.
 2020-03-15 22:07:22 -04:00
Andrew Gallant
b9cd95faf1 release: 12.0.0, take 2 2020-03-15 21:54:11 -04:00
Andrew Gallant
92daa34eb3 ripgrep: release 12.0.0 2020-03-15 21:42:54 -04:00
Andrew Gallant
a8c1fb7c88 changelog: prepare for 12.0.0 release 2020-03-15 21:06:45 -04:00
Andrew Gallant
52ec68799c ci: make script names consistent 2020-03-15 21:06:45 -04:00
Andrew Gallant
c0d78240df ci: remove Travis and appveyor specific stuff 2020-03-15 21:06:45 -04:00
Andrew Gallant
cda9acb876 ci: rebuild release infrastructure on GitHub Actions 2020-03-15 21:06:45 -04:00
Andrew Gallant
1ece50694e readme: update file size 2020-03-15 13:27:31 -04:00
Andrew Gallant
f3a966bcbc readme: add 'Unicode' label to ugrep 2020-03-15 13:26:02 -04:00
Andrew Gallant
a38913b63a readme: update benchmarks 
This also updates the corpora used, so previous times (and counts) are
not comparable.

We also remove some tools, likt pt, sift and ucg, since they appear to
be no longer maintained. ag isn't really maintained either, but it still
has significant mind share, so we retain a benchmark for it.

We also upgrade ack to version 3, and remove the clarification on how
`-w` is implemented.

We also add `git grep -P` (uses PCRE2) which appears to be much faster
than `git grep -E`.

Finally, we add ugrep which is a new up and comer in this space.

Fixes #1474
 2020-03-15 13:21:18 -04:00
Andrew Gallant
e772a95b58 regex: avoid using literal optimizations when whitespace is detected 
If a literal is entirely whitespace, then it's quite likely that it is
very common. So when that case occurs, just don't do (inner) literal
optimizations at all.

The regex engine may still make sub-optimal decisions here, but that's a
problem for another day.

Fixes #1087
 2020-03-15 13:19:14 -04:00
Andrew Gallant
9dd4bf8d7f style: fix rust-analyzer lint warnings 2020-03-15 13:19:14 -04:00
Andrew Gallant
c4c43c733e cli: add --no-ignore-files flag 
The purpose of this flag is to force ripgrep to ignore all --ignore-file
flags (whether they come before or after --no-ignore-files).

This flag can be overridden with --ignore-files.

Fixes #1466
 2020-03-15 13:19:14 -04:00
Andrew Gallant
447506ebe0 doc: clarify globing behavior 
Fixes #1442, Fixes #1478
 2020-03-15 13:19:14 -04:00
Andrew Gallant
12e4180985 doc: remove CPU features from man pages 
It doesn't really belong in the man page since it's an artifact of a
build/runtime configuration. Moreover, it inhibits reproducible builds.

Fixes #1441
 2020-03-15 13:19:14 -04:00
Andrew Gallant
daa8319398 doc: note ripgrep's stdin behavior 
Fixes #1439
 2020-03-15 13:19:14 -04:00
pierrenn
3a6a24a52a cli: add engine flag 
This permits switching between the different regex engine modes that
ripgrep supports. The purpose of this flag is to make it easier to
extend ripgrep with additional regex engines.

Closes #1488, Closes #1502
 2020-03-15 09:30:58 -04:00
pierrenn
aab3d80374 args: refactor to permit adding other engines 
This is in preparation for adding a new --engine flag which is intended
to eventually supplant --auto-hybrid-regex.

While there are no immediate plans to add more regex engines to ripgrep,
this is intended to make it easier to maintain a patch to ripgrep with
an additional regex engine. See #1488 for more details.
 2020-03-15 09:24:28 -04:00
Andrew Gallant
1856cda77b style: fix rust-analyzer lints in core 2020-03-15 09:04:54 -04:00
Andrew Gallant
7340d8dbbe deps: update everything 
This adds one new dependency, maybe-uninit, which is brought in by
crossbeam-channel[1]. This is to apparently fix some unsound code
without bumping the MSRV. Since ripgrep uses the latest stable release
of Rust, the maybe-uninit crate should compile down to nothing and just
re-export std's `MaybeUninit` type.

[1] - https://github.com/crossbeam-rs/crossbeam/pull/458
 2020-03-15 08:32:33 -04:00
chip
50d2047ae2 crates: update URLs in Cargo.toml 
This corrects an oversight when the repo was re-organized to
have its crates moved into a 'crates' sub-directory.

PR #1505
 2020-02-28 20:31:43 -05:00
Wolf Honore
227436624f ignore/types: add coq type 
PR #1504
 2020-02-28 19:11:29 -05:00
pierreN
5bfdd3a652 ci: fix ci by removing fetch-depth 1 
It's not clear why removing this makes things work. I've submitted
PRs that passed CI with fetch-depth=1. Maybe it only fails when
PRs are submitted from external contributors?

Either way, for now, we remove this and absorb the extra cost in
order to get PRs passing CI again.

PR #1501
 2020-02-27 08:53:06 -05:00
Andrew Gallant
ecec6147d1 doc: be more vague in the FAQ 
The existing vagueness was not enough to prevent people from lawyering
me over it.
 2020-02-22 09:13:31 -05:00
Lucien Greathouse
db7a8cdcb5 globset: Implement serde::{Serialize, Deserialize} for Glob 
PR #1492
 2020-02-21 07:40:47 -05:00
Andrew Gallant
eef7a7e7ff readme: update CI badge 2020-02-20 18:15:15 -05:00
Andrew Gallant
4176050cdd ignore: another simplification 
Again, thanks to @zsugabubus!
 2020-02-20 17:26:34 -05:00
Andrew Gallant
109460fce2 ignore: simplify parallel worker initialization 
We can just ask the channel whether any work has been loaded. Normally
querying a channel for its length is a strong predictor of bugs, but in
this case, we do it before we ever attempt a `recv`, so it should work.

Kudos to @zsugabubus for suggesting this!
 2020-02-20 16:50:41 -05:00
Andrew Gallant
da3431b478 ci: switch build to GitHub Actions 2020-02-20 16:07:51 -05:00
Andrew Gallant
f314b0d55f ignore: fix parallel traversal 
It turns out that the previous version wasn't quite correct. Namely, it
was possible for the following sequence to occur:

1. Consider that all workers, except for one, are `waiting`.
2. The last remaining worker finds one more job to do and sends it on
   the channel.
3. One of the previously `waiting` workers wakes up from the job that
   the last running worker sent, but `self.resume()` has not been
   called yet.
4. The last worker, from (2), calls `get_work` and sees that the
   channel has nothing on it, so it executes `self.waiting() ==
   1`. Since the worker in (3) hasn't called `self.resume()` yet,
   `self.waiting() == 1` evaluates to true.
5. This sets off a chain reaction that stops all workers, despite that
   fact that (3) got more work (which could itself spawn more work).

The end result is that the traversal may terminate while their are still
outstanding work items to process. This problem was observed through
spurious failures in CI. I was not actually able to reproduce the bug
locally.

We fix this by changing our strategy to detect termination using a
counter. Namely, we increment the counter just before sending new work
and decrement the counter just after finishing work. In this way, we
guarantee that the counter only ever reaches 0 once there is no more
work to process.

See #1337 for more discussion. Many thanks to @zsugabubus for helping me
work through this.
 2020-02-20 16:07:51 -05:00
Andrew Gallant
fab5c812f3 tests: add debugging output 
The transient failures appear to be persisting and they are quite
difficult to debug. So include a full directory listing in the output of
every test failure.
 2020-02-20 16:07:51 -05:00
Andrew Gallant
c824d095a7 tests: use std::env::consts::EXE_SUFFIX 
This avoids a conditional compilation knob and is likely more portable.
 2020-02-20 16:07:51 -05:00
Andrew Gallant
ee21897ebd tests: make 'cross test' work 
The reason why it wasn't working was the integration tests. Namely, the
integration tests attempted to execute the 'rg' binary directly from
inside cross's docker container. But this obviously doesn't work when
'rg' was compiled for a totally different architecture.

Cross normally does this by hooking into the Rust test infrastructure
and causing tests to run with 'qemu'. But our integration tests didn't
do that. This commit fixes our test setup to check for cross's
environment variable that points to the 'qemu' binary. Once we have
that, we just use 'qemu-foo rg' instead of 'rg'. Piece of cake.
 2020-02-20 16:07:51 -05:00
Andrew Gallant
0373f6ddb0 ci: soft-disable Travis and AppVeyor 2020-02-20 16:07:51 -05:00
asymmetric
b44554c803 ignore/types: add K type 
Adds support for files used by the K executable semantic framework:
http://www.kframework.org/index.php/Main_Page

PR #1493
 2020-02-19 07:07:09 -05:00
Andrew Gallant
0874aa115c repo: make ripgrep build with the new organization 2020-02-17 19:24:53 -05:00
Andrew Gallant
fdd8510fdd repo: move all source code in crates directory 
The top-level listing was just getting a bit too long for my taste. So
put all of the code in one directory and shrink the large top-level mess
to a small top-level mess.

NOTE: This commit only contains renames. The subsequent commit will
actually make ripgrep build again. We do it this way with the naive hope
that this will make it easier for git history to track the renames.
Sigh.
 2020-02-17 19:24:53 -05:00
Andrew Gallant
0bc4f0447b style: rustfmt everything 
This is why I was so intent on clearing the PR queue. This will
effectively invalidate all existing patches, so I wanted to start from a
clean slate.

We do make one little tweak: we put the default type definitions in
their own file and tell rustfmt to keep its grubby mits off of it. We
also sort it lexicographically and hopefully will enforce that from here
on.
 2020-02-17 19:24:53 -05:00
Andrew Gallant
c95f29e3ba ci: check rustfmt in Travis 2020-02-17 19:24:53 -05:00
Andrew Gallant
3644208b03 ci: set MSRV to Rust 1.41.0 
The next release will be ripgrep 12, so we bump to the latest stable
release of Rust.
 2020-02-17 19:24:53 -05:00
Andrew Gallant
66f045e055 changelog: add commit links 
... now that we have stable identifiers.
 2020-02-17 17:34:19 -05:00
zsugabubus
3d59bd98aa ignore: rework inter-thread messaging 
Change the meaning of `Quit` message. Now it means terminate. The final
"dance" is unnecessary, because by the time quitting begins, no thread
will ever spawn a new `Work`. The trick was to replace the heuristic
spin-loop with blocking receive.

Closes #1337
 2020-02-17 17:16:28 -05:00
Andrew Gallant
52d7f47420 ignore: treat symbolic links to directories as directories 
Due to how walkdir works if symlinks are not followed, symlinks to
directories are seen as simple files by ripgrep. This caused a panic
in some cases due to receiving a WalkEvent::Exit event without a
corresponding WalkEvent::Dir event.

This is fixed by looking at the metadata of the file in the case of a
symlink to determine if it's a directory. We are careful to only do
this stat check when the depth of the entry is 0, as this bug only
impacts us when 1) we aren't following symlinks generally and 2) the
user provides a symlinked directory that we do follow as a top-level
path to search.

Fixes #1389, Closes #1397
 2020-02-17 17:16:28 -05:00
Andrew Gallant
75cbe88fa2 cli: add --no-unicode, deprecate --no-pcre2-unicode 
This adds a universal --no-unicode flag that is intended to work for all
supported regex engines. There is no point in retaining
--no-pcre2-unicode, so we make them aliases to the new flags and
deprecate them.
 2020-02-17 17:16:28 -05:00
Andrew Gallant
711426a632 cli: add --no-require-git flag 
This flag prevents ripgrep from requiring one to search a git repository
in order to respect git-related ignore rules (global, .gitignore and
local excludes). This actually corresponds to behavior ripgrep had long
ago, but #934 changed that. It turns out that users were relying on this
buggy behavior. In most cases, fixing it as simple as converting one's
rules to .ignore or .rgignore files. Unfortunately, there are other use
cases---like Perforce automatically respecting .gitignore files---that
make a strong case for ripgrep to at least support this.

The UX of a flag like this is absolutely atrocious. It's so obscure that
it's really not worth explicitly calling it out anywhere. Moreover, the
error cases that occur when this flag isn't used (but its behavior is
desirable) will not be intuitive, do not seem easily detectable and will
not guide users to this flag. Nevertheless, the motivation for this is
just barely strong enough for me to begrudgingly accept this.

Fixes #1414, Closes #1416
 2020-02-17 17:16:28 -05:00
Andrew Gallant
01eeec56bb deb: fix fish completion install location 
It looks like `completions` is owned by Fish itself. Third party
completions should go in `vendor_completions.d`.

Fixes #1485
 2020-02-17 17:16:28 -05:00
Jakub Wieczorek
322fc75a3d ignore: make walker visit untraversable directories 
This commit fixes an inconsistency between the serial and the parallel
directory walkers around visiting a directory for which the user holds
insufficient permissions to descend into.

The serial walker does produce a successful entry for a directory that
it cannot descend into due to insufficient permissions. However, before
this change that has not been the case for the parallel walker, which
would produce an `Err` item not only when descending into a directory
that it cannot read from but also for the directory entry itself.

This change brings the behaviour of the parallel variant in line with
that of the serial one.

Fixes #1346, Closes #1365
 2020-02-17 17:16:28 -05:00
Jakub Wieczorek
b435eaafc8 grep-regex: fix inner literal extraction bug 
This appears to be another transcription bug from copying this code from
the prefix literal detection from inside the regex crate. Namely, when
it comes to inner literals, we only want to treat counted repetition as
two separate cases: the case when the minimum match is 0 and the case
when the minimum match is more than 0. In the former case, we treat
`e{0,n}` as `e*` and in the latter we treat `e{m,n}` where `m >= 1` as
just `e`.

We could definitely do better here. e.g., This means regexes like
`(foo){10}` will only have `foo` extracted as a literal, where searching
for the full literal would likely be faster.

The actual bug here was that we were not implementing this logic
correctly. Namely, we weren't always "cutting" the literals in the
second case to prevent them from being expanded.

Fixes #1319, Closes #1367
 2020-02-17 17:16:28 -05:00
Ed Page
f8e70294d5 ignore: allow post-processing at end-of-thread 
On top of the parallel-walk's closures, this provides a Visitor API.
This clarifies the role of the two different closures in the `run`
API and allows implementing of `Drop` for post-processing once traversal
is finished.

The closure API is maintained not just for compatibility but also
convinience for simple cases.

Fixes #469, Closes #1430
 2020-02-17 17:16:28 -05:00
Ed Page
578e2d47a8 core: simplify parallel walking using borrows 
This changes ripgrep to use ignore's new support for borrowing data when
walking in parallel.
 2020-02-17 17:16:28 -05:00
Ed Page
9f7c2ebc09 ignore: allow parallel walker to borrow data 
This makes it so the caller can more easily refactor from
single-threaded to multi-threaded walking. If they want to support both,
this makes it easier to do so with a single initialization code-path. In
particular, it side-steps the need to put everything into an `Arc`.

This is not a breaking change because it strictly increases the number
of allowed inputs to `WalkParallel::run`.

Closes #1410, Closes #1432
 2020-02-17 17:16:28 -05:00
Andrew Gallant
5c1eac41a3 changelog: highlight a bad performance regression 2020-02-17 17:16:28 -05:00
Johannes Altmanninger
6f2b79f584 ignore: use git commondir for sourcing .git/info/exclude 
Git looks for this file in GIT_COMMON_DIR, which is usually the same
as GIT_DIR (.git). However, when searching inside a linked worktree,
.git is usually a file that contains the path of the actual git dir,
which in turn contains a file "commondir" which references the directory
where info/exclude may reside, alongside other configuration shared across
all worktrees. This directory is usually the git dir of the main worktree.

Unlike git this does *not* read environment variables GIT_DIR and
GIT_COMMON_DIR, because it is not clear how to interpret them when
searching multiple repositories.

Fixes #1445, Closes #1446
 2020-02-17 17:16:28 -05:00
Andrew Gallant
0c3b673e4c cli: make ripgrep work in non-existent directories 
It turns out that querying the CWD while in a directory that no longer
exists results in an error. Since the CWD is queried every time ripgrep
starts---whether it needs it or not---for dealing with glob matching,
ripgrep winds up being completely useless inside a non-existent
directory.

We fix this in a few different ways:

* Firstly, if std::env::current_dir() fails, then we fall back to trying
  to read the `PWD` environment variable.
* If that fails, that we return a more sensible error message so that a
  user can at least react to the problem. Previously, the error message
  was inscrutable.
* Finally, we try to avoid the problem altogether by building empty glob
  matchers if not globs were provided, thus side-stepping querying the
  CWD completely.

Fixes #1291, Closes #1400
 2020-02-17 17:16:28 -05:00
Naveen Nathan
297b428c8c cli: add --no-ignore-exclude flag 
This commit adds a new --no-ignore-exclude flag that permits disabling
the use of .git/info/exclude filtering. Local exclusions are manual
configurations to a repository and are not shared, so it is sometimes
useful to disable to get a consistent view of a repository.

This also adds a new section to the man page that describes automatic
filtering.

Closes #1420
 2020-02-17 17:16:28 -05:00
Manfred Endres
804b43ecd8 globset: implement FromStr for Glob 
The `globset::Glob` type [`new`] function creates a new value with an
`&str` parameter which returns an `Result<Glob, Error>` object. This is
exactly what [`std::str::FromStr::from_str`][`std::str::FromStr`] defines.
Libraries like [`clap`] use [`std::str::FromStr`] to create objects from
provided commandline arguments. This change makes this library usable
without a newtype wrapper.

[`std::str::FromStr`]: 	https://doc.rust-lang.org/std/str/trait.FromStr.html
[`clap`]:		https://docs.rs/clap/2.33.0/clap/macro.value_t.html
[`new`]:		https://docs.rs/globset/0.4.4/globset/struct.Glob.html#method.new

Closes #1447
 2020-02-17 17:16:28 -05:00
Lucien Greathouse
2263b8ac92 globset: add GlobMatcher::glob 
This exposes the underlying `Glob` used to compile the matcher. This can
be useful for wrapping up the glob matcher in other types.

Closes #1454
 2020-02-17 17:16:28 -05:00
Andrew Gallant
cd8ec38a68 grep-regex: add fast path for -w/--word-regexp 
Previously, ripgrep would always defer to the regex engine's capturing
matches in order to implement word matching. Namely, ripgrep would
determine the correct match offsets via a capturing group, since the
word regex is itself generated from the user supplied regex.

Unfortunately, the regex engine's capturing mode is still fairly slow,
so this commit adds a fast path to avoid capturing mode in the vast
majority of cases. See comments in the code for details.
 2020-02-17 17:16:28 -05:00
Andrew Gallant
6a0e0147e0 grep-regex: improve literal detection with -w 
When the -w/--word-regexp was used, ripgrep would in many cases fail to
apply literal optimizations. This occurs specifically when the regex
given by the user is an alternation of literals with no common prefixes
or suffixes, e.g.,

    rg -w 'foo|bar|baz|quux'

In this case, the inner literal detector fails. Normally, this would
result in literal prefixes being detected by the regex engine. But
because of the -w/--word-regexp flag, the actual regex that we run ends
up looking like this:

    (^|\W)(foo|bar|baz|quux)($|\W)

which of course defeats any prefix or suffix literal optimizations in
the regex crate's somewhat naive extractor. (A better extractor could
still do literal optimizations in the above case.)

So this commit fixes this by falling back to prefix or suffix literals
when they're available instead of prematurely giving up and assuming the
regex engine will do the rest.
 2020-02-17 17:16:28 -05:00
Andrew Gallant
ad97e9c93f grep-regex: improve inner literal detection 
This fixes an interesting performance bug where the inner literal
extractor would sometimes choose a sub-optimal literal. For example,
consider the regex:

    \x20+Sherlock Holmes\x20+

(The `\x20` is the ASCII code for a space character, which we use here
to just make it clearer. It otherwise does not matter.)

Previously, this would see the initial \x20 and then stop collecting
literals after the `+` repetition operator. This was because the inner
literal detector was adapter from the prefix literal detector, which had
to stop here. Namely, while \x20S would be a valid prefix (for example),
\x20\x20S would also be a valid prefix. As would \x20\x20\x20S and so
on. So the prefix detector would have to stop at the repetition
operator. Otherwise, only searching for \x20S could potentially scan
farther then the starting position of the next match.

However, for inner literals, this calculus no longer makes sense. We can
freely search for, e.g., \x20S without missing matches that start with
\x20\x20S precisely because we know this is an inner literal which may
not correspond to the start of a match.

With this fix, the literal that is now detected is

    \x20Sherlock Holmes\x20

Which is much better. We achieve this by no longer "cutting" literals
after seeing a `+` repetition operator. Instead, we permit literals to
continue to be extended.

The reason why this is important is because using \x20 as the literal to
search for is generally bad juju since it is so common. In fact, we
should probably add more logic here to either avoid such things or give
up entirely on the inner literal optimization if it detected a literal
that we think is very common. But we punt on such things here.
 2020-02-17 17:16:28 -05:00
Robert Irelan
24f8a3e5ec doc: document all file type 
This adds it to the guide and the docs for the --type flag.

Fixes #1344, Closes #1472
 2020-02-17 17:16:28 -05:00
Mikko Vedru
1bdb767851 doc: improve docs for --sort and --sortr flags 
I improved the help documentation in the following manner and for the
following reasons:

1. It's only logical to put the default sub-option on the first possible
line, as well as to separately mention that it is indeed the default
sub-option.

2. Additional options for the flags should describe the main points of
their purpose without requiring user to read the whole help entry. In my
opinion, the information sub-options' influence on multi-threading and
speed are important enough to warrant their inclusion in each
sub-option's description line text.

Closes #1434
 2020-02-17 17:16:28 -05:00
Andreas Stieger
a4897eca23 readme: simplify openSUSE instructions 
Closes #1436
 2020-02-17 17:16:28 -05:00
Collin Styles
a070722ff2 cli: add --include-zero flag 
This flag, when used in conjunction with --count or --count-matches,
will print a result for each file searched even if there were zero
matches in that file. This is off by default but can be enabled to make
ripgrep behave more like grep.

This also clarifies some of the defaults for the
grep-printer::SummaryBuilder type.

Closes #1370, Closes #1405
 2020-02-17 17:16:28 -05:00
Matěj Cepl
4628d77808 ignore/types: add spec file type 
This is for RPM package SPEC files.

Fixes #946, Closes #1449
 2020-02-17 17:16:28 -05:00
Ximin Luo
f8418c6a52 explicitly declare lazy_static dependency 
`benches/bench.rs` uses lazy_static but Cargo.toml does not declare a
dependency on it. This causes rustc to use its own internal private
copy instead. Sometimes this causes unintuitive errors like this Debian
bug: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=942243

The underlying issue is https://github.com/rust-lang/rust#27812 but it
can be avoided by explicitly declaring the dependency, which you are
supposed to do anyways.

Closes #1435
 2020-02-17 17:16:28 -05:00
luh2
040ca45ba0 ignore/types: add xhtml to xml file type 
Closes #1426
 2020-02-17 17:16:28 -05:00
Andrew Gallant
91470572cd changelog: add notes about new file types 2020-02-17 17:16:28 -05:00
Sven-Hendrik Haase
027adbf485 ignore/types: add 'diff' file type 
This includes .patch and .diff files.

Fixes #1418, Closes #1419
 2020-02-17 17:16:28 -05:00
Mohammad AlSaleh
e71eedf0eb cli: add --no-context-separator flag 
--context-separator='' still adds a new line separator, which could
still potentially be useful. So we add a new `--no-context-separator`
flag that completely disables context separators even when the -A/-B/-C
context flags are used.

Closes #1390
 2020-02-17 17:16:28 -05:00
Andrew Gallant
88f46d12f1 tests: remove existing test directory 
I'm surprised this wasn't caught until now, but if a test directory
already exists, then it was reused. This can result in hard to debug
problems with tests when, e.g., file names are changed and a recursive
search is executed.
 2020-02-17 17:16:28 -05:00
sharkdp
a18cf6ec39 ignore: add existence check for ignore files 
This commit adds a simple `.exists()` check for `.gitignore`,
`.ignore`, and other similar files before actually calling
`File::open(…)` in `GitIgnoreBuilder::add`.

The reason is that a simple existence check via `stat` can be faster
than actually trying to `open` the file, see
https://stackoverflow.com/a/12774387/704831. As we typically expect(?)
the number of directories *without* ignore files to be much larger
than the number of directories *with* ignore files, this leads to an
overall speedup.

The performance gain is not huge for `rg`, but can be quite significant
if more `.gitignore`-like files are added via
`add_custom_ignore_filename`. The speedup is *larger* for folders with
*low* files-per-directory ratios.

Note though that we do not do this check on Windows until a specific
analysis there suggests this is beneficial. Namely, Windows generally
has slower file system operations, so it's not clear whether this
speculative check is actually a benefit or not.

Benchmark results
-----------------

`rg --files` in my home folder (200k results, 6.5 files per directory):

| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|---:|---:|---:|---:|
| `./rg-master --files` | 396.4 ± 3.2 | 390.9 | 400.0 | 1.05 |
| `./rg-feature --files` | 376.0 ± 3.6 | 369.3 | 383.5 | 1.00 |

`rg --files --hidden` in my home folder (800k results, 5.4
files per directory)

| Command | Mean [s] | Min [s] | Max [s] | Relative |
|:---|---:|---:|---:|---:|
| `./rg-master --files --hidden` | 1.575 ± 0.012 | 1.560 | 1.597 | 1.06 |
| `./rg-feature --files --hidden` | 1.479 ± 0.011 | 1.464 | 1.496 | 1.00 |

`rg --files` in the chromium-79.0.3915.2 source tree (300k results, 12.7 files per
directory)

| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|---:|---:|---:|---:|
| `~/rg-master --files` | 445.2 ± 5.3 | 435.6 | 453.0 | 1.04 |
| `~/rg-feature --files` | 428.9 ± 7.0 | 418.2 | 440.0 | 1.00 |

`rg --files` in the linux-5.3 source tree (65k results, 15.1
files per directory)

| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
|:---|---:|---:|---:|---:|
| `./rg-master --files` | 94.5 ± 1.9 | 89.8 | 98.5 | 1.02 |
| `./rg-feature --files` | 92.6 ± 2.7 | 88.4 | 98.7 | 1.00 |

Closes #1381
 2020-02-17 17:16:28 -05:00
Gibson Fahnestock
c78c3236a8 readme: remove outdated SIMD info 
Looks like the upstream brew Formula [0][] now has SIMD support, so
remove the extraneous info now that the custom tap is no longer needed
[1][].

[0]: https://github.com/Homebrew/homebrew-core/blob/master/Formula/ripgrep.rb
[1]: f3083e4574

PR #1431
 2020-02-15 17:19:22 -05:00
Sorin Sbarnea
7cf21600cd readme: document CentOS 8 support 
ripgrep install instructions are valid even for the 7 version. The tool
works without problems on these too.

PR #1428
 2020-02-15 17:16:57 -05:00
Jonathan Mast
647b0d3977 ignore/types: add HAML and ERB 
These are commonly used templating languages for Ruby, add their
extensions to the filetypes list for convenient filtering.

PR #1407
 2020-02-15 09:18:32 -05:00
Jeff S
e572fc1683 ignore/types: add slim, slime, and skim templates 
PR #1391
 2020-02-15 09:17:46 -05:00
Andrew Gallant
9cb93abd11 ignore: allow use of Error::description 
We can remove it in the next semver incompatible release.
 2020-02-10 06:44:21 -05:00
Luca Kredel
41695c66fa ignore/types: add typoscript file type 
Add the file types for TypoScript - the configuration language of the
TYPO3 CMS.

PR #1477
 2020-02-07 08:41:00 -05:00
Andrew Gallant
cb0dfda936 faq: add section about donations 
This is asked often enough that it's worth having a canonical answer.
 2020-02-05 13:09:11 -05:00
Andrew Gallant
74d1fe59e9 deps: update everything 2020-01-30 18:33:40 -05:00
Andrew Gallant
9fd1e202e0 deps: update regex, regex-syntax and aho-corasick 
Notably, this brings in a bug fix reported by @okdana:
https://github.com/rust-lang/regex/issues/640
 2020-01-30 18:32:56 -05:00
Robert Irelan
e76807b1b5 ignore/types: add *.org_archive to org file type 
.org_archive is the default extension for Org archive files, created when
entries from an Org-mode file are archived (see
<https://orgmode.org/org.html#Moving-subtrees>). These files are still in Org
mode format, so it's worth searching them at the same time as non-archive Org
mode files.

PR #1475
 2020-01-29 13:59:34 -05:00
Andrew Gallant
f8fb65f7e3 globset: fix benchmarks 
There were apparently a lot of unused things, including lazy_static.
 2020-01-27 16:45:12 -05:00
Tristan Waddington
98de8d248a ignore/types: make 'gradle' it's own type 
This change maintains the existing behavior of the 'groovy' type, which
includes both .groovy and .gradle files.

PR #1470
 2020-01-23 06:51:11 -05:00
Crestwave
c358700dfb readme: add instructions for Haiku x86_64 and x86_gcc2 
PR #1465
 2020-01-21 07:34:24 -05:00
Alex Touchet
8670a4a969 readme: update outdated links 
PR #1463
 2020-01-21 07:32:54 -05:00
Oliver Newman
e3b1f86908 doc: add missing "will" to the user guide 
PR #1462
 2020-01-20 17:26:08 -05:00
Jan Verbeek
46b07bb2ee ignore/types: fix postscript globs 
The postscript globs were missing asterisks, so they were treated as
literal filenames.

PR #1461
 2020-01-20 07:48:57 -05:00
Andrew Gallant
8bdf84e3a8 deps: update everything 2020-01-16 19:47:23 -05:00
Andrew Gallant
5a6e17fcc1 deps: various updates 
Most of these updates (sans thread_local) are from crates I maintain
that have seen updates recently.

Notably, this includes a bump to `termcolor 1.1.0` which includes
support for respecting `NO_COLOR`. This commit therefore means that
ripgrep now supports `NO_COLOR`.

As an added bonus, we drop a dependency on Windows. (Although the total
amount of code compiled remains the same.)

Closes #1186
 2020-01-11 10:09:10 -05:00
Andrew Gallant
00bfcd14a6 ignore-0.4.11 2020-01-10 15:08:27 -05:00
Andrew Gallant
bf0ddc4675 ci: fix musl docker build 
Looks like the old japaric images are bunk. We update our docker image
to be based on the new rustembedded images and configure cross to use
it.

Turns out that this wasn't due to a stale docker image, but rather, a
bug in cross: https://github.com/rust-embedded/cross/issues/357
We work around that bug by installing the master branch of cross. Sigh.
 2020-01-10 15:07:47 -05:00
Andrew Gallant
0fb3f6a159 ci: disable github actions for now 
The CI build failures are annoying and distracting. Hopefully soon I'll
be able to invest more time in the switch.
 2020-01-10 15:07:47 -05:00
Andrew Gallant
837fb5e21f deps: update to crossbeam-channel 0.4 
Closes #1427
 2020-01-10 15:07:47 -05:00
Andrew Gallant
2e1815606e deps: update to bytecount 0.6 
Looks like there aren't any major changes other than dependency updates.
 2020-01-10 15:07:47 -05:00
Andrew Gallant
cb2f6ddc61 deps: update to thread_local 1.0 
We also update the pcre2 and regex dependencies, which removes any other
lingering uses of thread_local 0.3.
 2020-01-10 15:07:47 -05:00
Andrew Gallant
bd7a42602f deps: bump to base64 0.11 2020-01-10 15:07:47 -05:00
Andrew Gallant
528ce56e1b deps: run cargo update 
The only new dependency is an unused target specific dependency hermit
via the atty crate.
 2020-01-10 15:07:47 -05:00
Yevgen Antymyrov
8892bf648c doc: fix typo in FAQ 2019-09-25 08:13:27 -04:00
Jonathan Clem
8cb7271b64 ci: get GitHub Actions running again 
Basically, matrix.os needs to be defined for every build. We
were commenting out some of the builds in order to debug
CI in the `include` section, but we also need to comment them
out in the `build section.
 2019-09-11 09:08:24 -04:00
Andrew Gallant
4858267f3b ci: initial github actions config 2019-08-31 09:24:44 -04:00
Andrew Gallant
5011dba2fd ignore: remove unused parameter 2019-08-28 20:21:34 -04:00
Andrew Gallant
e14f9195e5 deps: update everything 2019-08-28 20:18:47 -04:00
Andrew Gallant
ef0e7af56a deps: update bstr to 0.2.7 
The new bstr release contains a small performance bug fix where some
trivial methods weren't being inlined.
 2019-08-11 10:41:05 -04:00
Todd Walton
b266818aa5 doc: use XDG_CONFIG_HOME in comments 
XDG_CONFIG_DIR does not actually exist.

PR #1347
 2019-08-09 13:37:37 -04:00
LawAbidingCactus
81415ae52d doc: update to reflect glob matching behavior change 
Specifically, paths contains a `/` are not allowed to match any
other slash in the path, even as a prefix. So `!.git` is the correct
incantation for ignoring a `.git` directory that occurs anywhere 
in the path.
 2019-08-07 13:47:18 -04:00
Andrew Gallant
5c4584aa7c grep-regex-0.1.5 2019-08-06 09:51:13 -04:00
Andrew Gallant
0972c6e7c7 grep-searcher-0.1.6 2019-08-06 09:50:52 -04:00
Andrew Gallant
0a372bf2e4 deps: update ignore 2019-08-06 09:50:35 -04:00
Andrew Gallant
345124a7fa ignore-0.4.10 2019-08-06 09:47:45 -04:00
Andrew Gallant
31807f805a deps: drop tempfile 
We were only using it to create temporary directories for `ignore`
tests, but it pulls in a bunch of dependencies and we don't really need
randomness. So just use our own simple wrapper instead.
 2019-08-06 09:46:05 -04:00
Andrew Gallant
4de227fd9a deps: update everything 
Mostly this just updates regex and its assorted dependencies. This does
drop utf8-ranges and ucd-util, in accordance with changes to
regex-syntax and regex.
 2019-08-05 13:50:55 -04:00
jimbo1qaz
d7ce274722 readme: Debian Buster is stable now 
PR #1338
 2019-08-04 08:06:10 -04:00
Andrew Gallant
5b10328f41 changelog: update with bug fix 2019-08-02 07:37:27 -04:00
Andrew Gallant
813c676eca searcher: fix roll buffer bug 
This commit fixes a subtle bug in how the line buffer was rolling its
contents. Specifically, when ripgrep searches without memory maps,
it uses a "roll" buffer for incremental line oriented search without
needing to read the entire file into memory at once. The roll buffer
works by reading a chunk of bytes from the file into memory, and then
searching everything in that buffer up to the last `\n` byte. The bytes
*after* the last `\n` byte are preserved, since they likely correspond
to *part* of the next line. Once ripgrep is done searching the buffer,
it "rolls" the buffer such that the start of the next line is at the
beginning of the buffer, and then ripgrep reads more data into the
buffer starting at the (possibly) partial end of that line.

The implication of this strategy, necessarily so, is that a buffer must
be big enough to fit a single line in memory. This is because the regex
engine needs a contiguous block of memory to search, so there is no way
to search anything smaller than a single line. So if a file contains a
single line with 7.5 million bytes, then the buffer will grow to be at
least that size. (Many files have super long lines like this, but they
tend to be *binary* files, which ripgrep will detect and stop searching
unless the user forces it with the `-a/--text` flag. So in practice,
they aren't usually a problem. However, in this case, #1335 found a case
where a plain text file had a line with 7.5 million bytes.)

Now, for performance reasons, ripgrep reuses these buffers across its
search. Typically, it will create `N` of these line buffers when it
starts (where `N` is the number of threads it is using), and then reuse
them without creating any new ones as it searches through files.

This means that if you search a file with a very long line, that buffer
will expand to be big enough to store that line. ripgrep never contracts
these buffers, so once it searches the next file, ripgrep will continue
to use this large buffer. While it might be prudent to contract these
buffers in some circumstances, this isn't otherwise inherently a
problem. The memory has already been allocated, and there isn't much
cost to using it, other than the fact that ripgrep hangs on to it and
never gives it back to the OS.

However, the `roll` implementation described above had a really
important bug in it that was impacted by the size of the buffer.
Specifically, it used the following to "roll" the partial line at the
end of the buffer to the beginning:

    self.buf.copy_within_str(self.pos.., 0);

Which means that if the buffer is very large, ripgrep will copy
*everything* from `self.pos` (which might be very small, e.g., for small
files) to the end of the buffer, and move it to the beginning of the
buffer. This will happen repeatedly each time the buffer is used to
search small files, which winds up being quite a large slow down if the
line was exceptionally large (say, megabytes).

It turns out that copying everything is completely unnecessary. We only
need to copy the remainder of the last read to the beginning of the
buffer. Everything *after* the last read in the buffer is just free
space that can be filled for the next read. So, all we need to do is
copy just those bytes:

    self.buf.copy_within_str(self.pos..self.end, 0);

... which is typically much much smaller than the rest of the buffer.

This was likely also causing small performance losses in other cases as
well. For example, when searching a lot of small files, ripgrep would
likely do a lot more copying than necessary. Although, given that the
default buffer size is 8KB, this extra copying was likely pretty small,
and was thus harder to observe.

Fixes #1335
 2019-08-02 07:23:27 -04:00
Andrew Gallant
f625d72b6f pkg: update brew tap to 11.0.2 2019-08-01 19:39:53 -04:00
Andrew Gallant
3de31f7527 ci: fix musl deployment 
The docker image that the Linux binary is now built in does not have
ASCII doc installed, so setup Cross to point to my own image with those
tools installed.
 2019-08-01 18:41:44 -04:00
Andrew Gallant
e402d6c260 ripgrep: release 11.0.2 2019-08-01 18:02:15 -04:00
Andrew Gallant
48b5bdc441 src: remove old directories 
termcolor has had its own repository for a while now. No need for these
redirects any more.
 2019-08-01 17:49:28 -04:00
Andrew Gallant
709ca91f50 ignore: release 0.4.9 2019-08-01 17:48:37 -04:00
Andrew Gallant
9c220f9a9b grep-regex: release 0.1.4 2019-08-01 17:47:45 -04:00
Andrew Gallant
9085bed139 grep-matcher: release 0.1.3 2019-08-01 17:46:59 -04:00
Andrew Gallant
931ab35f76 changelog: start work on 11.0.2 release 2019-08-01 17:42:38 -04:00
Andrew Gallant
b5e5979ff1 deps: update everything 
This drops `spin` and `autocfg`, yay.
 2019-08-01 17:42:38 -04:00
Andrew Gallant
052c857da0 doc: mention .ignore and .rgignore more prominently 
Fixes #1284
 2019-08-01 17:37:46 -04:00
Andrew Gallant
5e84e784c8 doc: add translations section 
We note that they may not be up to date and are unofficial.

Fixes #1246
 2019-08-01 17:37:46 -04:00
Andrew Gallant
01e8e11621 doc: improve PCRE2 failure mode documentation 
If a user tries to search for an explicit `\n` character in a PCRE2
regex, ripgrep won't report an error and instead will (likely) silently
fail to match.

Fixes #1261
 2019-08-01 17:32:44 -04:00
Ninan John
9268ff8e8d ripgrep: fix bug when CWD has directory named - 
Specifically, when searching stdin, if the current directory has a
directory named `-`, then the `--with-filename` flag would automatically
be turned on. This is because `--with-filename` is automatically enabled
when ripgrep is given a single path that is a directory. When ripgrep is
given empty arguments, and if it is searching stdin, then its default
path list is just simple `["-"]`. The `is_dir` check passes, and
`--with-filename` gets enabled.

This commit fixes the problem by checking whether the path is `-` first.
If so, then we assume it isn't a directory. This is fine, since if it is
a directory and one asks to search it explicitly, then ripgrep will
interpret `-` as stdin anyway (which is arguably a bug on its own, but
probably not one worth fixing).

Fixes #1223, Closes #1292
 2019-08-01 17:27:23 -04:00
dana
c2cb0a4de4 ripgrep: add --glob-case-insensitive 
This flag forces -g/--glob patterns to be treated case-insensitively, as with
--iglob patterns.

Fixes #1293
 2019-08-01 17:08:58 -04:00
Andrew Gallant
adb9332f52 regex: fix -F aho-corasick optimization 
It turns out that when the -F flag was used, if any of the patterns
contained a regex meta character (such as `.`), then we winded up
escaping the pattern first before handing it off to Aho-Corasick, which
treats all patterns literally.

We continue to apply band-aides here and just avoid Aho-Corasick if
there is an escape in any of the literal patterns. This is unfortunate,
but making this work better requires more refactoring, and the right
solution is to get this optimization pushed down into the regex engine.

Fixes #1334
 2019-08-01 16:58:12 -04:00
Matthew Davidson
bc37c32717 ignore/types: add edn type from Clojure ecosystem 
PR #1330
 2019-07-29 16:43:28 -04:00
Andrew Gallant
08ae4da2b7 deps: update them 
There are some nice removals. It looks like rand has slimmed down, and
smallvec is gone now as well.
 2019-07-25 07:52:33 -04:00
Andrew Gallant
7ac95c1f50 deps: bump ignore 2019-07-24 12:56:47 -04:00
Andrew Gallant
7a6903bd4e ignore-0.4.8 2019-07-24 12:56:01 -04:00
Tiziano Santoro
9801fae29f ignore: support compilation on wasm 
Currently the crate assumes that exactly one of `cfg(windows)` or
`cfg(unix)` is true, but this is not actually the case, for instance
when compiling for `wasm32`.

Implement the missing functions so that the crate can compile on other
platforms, even though those functions will always return an error.

PR #1327
 2019-07-24 12:55:37 -04:00
Miloš Stojanović
abdf7140d7 readme: fix broken link to Scoop bucket 
PR #1324
 2019-07-20 12:03:46 -04:00
Conrad Olega
b83e7968ef ignore/types: add Robot Framework 
PR #1322
 2019-07-14 08:12:34 -04:00
Hugo Locurcio
8ebc113847 doc: improve docs for --replace flag 
Specifically, we document shell-specific caveats related to the `--replace`
flag.

PR #1318
 2019-07-04 11:42:35 -04:00
Andrew Gallant
785c1f1766 release: globset, grep-cli, grep-printer, grep-searcher 2019-06-26 16:53:30 -04:00
Andrew Gallant
8b734cb490 deps: update everything 2019-06-26 16:51:06 -04:00
Andrew Gallant
b93762ea7a bstr: update everything to bstr 0.2 2019-06-26 16:47:33 -04:00
Andrew Gallant
34677d2622 search: a few small touchups 2019-06-18 20:23:47 -04:00
Andrew Gallant
d1389db2e3 search: better errors for preprocessor commands 
If a preprocessor command could not be started, we now show some
additional context with the error message. Previously, it showed
something like this:

  some/file: No such file or directory (os error 2)

Which is itself pretty misleading. Now it shows:

  some/file: preprocessor command could not start: '"nonexist" "some/file"': No such file or directory (os error 2)

Fixes #1302
 2019-06-16 19:02:02 -04:00
Andrew Gallant
50bcb7409e deps: update everything 2019-06-16 18:38:45 -04:00
Andrew Gallant
7b9972c308 style: fix deprecations 
Use `dyn` for trait objects and use `..=` for inclusive ranges.
 2019-06-16 18:37:51 -04:00
Hitesh Jasani
9f000c2910 ignore/types: add more nim types 
PR #1297
 2019-06-12 14:02:28 -04:00
skierpage
392682d352 doc: point regex doc link to the latest version 
The latest doc is different, e.g. adds "symmetric differences" under
https://docs.rs/regex/*/regex/#character-classes

PR #1287
 2019-06-01 08:44:55 -04:00
Andrew Gallant
7d3f794588 ignore: remove .git check in some cases 
When we know we aren't going to process gitignores, we shouldn't waste
the syscall in every directory to check for a git repo.
 2019-05-29 18:06:11 -04:00
bruce-one
290fd2a7b6 readme: mention Zstandard and Brotli 
Also alphabetise the list.

PR #1288
 2019-05-29 13:37:31 -04:00
Fabian Würfl
d1e4d28f30 readme: remove outdated statement 
Issue #10 already states that "ripgrep is now in most or all of the major
package repositories."

PR #1280
 2019-05-14 18:44:50 -04:00
Andrew Gallant
5ce2d7351d ci: use cross for musl x86_64 builds 
This is necessary because jemalloc + musl + Ubuntu 16.04 is apparently
broken.

Moreover, jemalloc doesn't support i686, so we accept the performance
regression there.

See also: https://github.com/gnzlbg/jemallocator/issues/124
 2019-04-25 11:12:14 -04:00
Andrew Gallant
9dcfd9a205 deps: bump pcre2-sys to 0.2.1 
This brings in a bug fix that no longer tries to run `git` to update the
submodule if the `git` command doesn't exist.

This is useful is more restricted build contexts where `git` isn't
installed. Such as in the docker image used for running `cross`.
 2019-04-25 11:12:14 -04:00
Andrew Gallant
36b276c6d0 printer: remove unnecessary mut 2019-04-24 17:22:27 -04:00
Andrew Gallant
03bf37ff4a alloc: use jemalloc when building with musl 
It turns out that musl's allocator is slow enough to cause a fairly
noticeable performance regression when ripgrep is built as a static
binary with musl. We fix this by using jemalloc when building with musl.

We continue to use the default system allocator in all other scenarios.
Namely, glibc's allocator doesn't noticeably regress performance compared
to jemalloc. But we could add more targets to this logic if other
system allocators (macOS, Windows) prove to be slow.

This wasn't necessary before because rustc recently stopped using jemalloc
by default.

Fixes #1268
 2019-04-24 17:21:38 -04:00
Andrew Gallant
e7829c05d3 cli: fix bug where last byte was stripped 
In an effort to strip line terminators, we assumed their existence. But
a pattern file may not end with a line terminator, so we shouldn't
unconditionally strip them.

We fix this by moving to bstr's line handling, which does this for us
automatically.
 2019-04-19 07:11:44 -04:00
Rory O’Kane
a6222939f9 readme: mention --pcre2 as long form of -P 
This is for consistency with the short and long flags given in other
bullet points. I originally assumed there was no long flag for `-P`
because none was given here.

PR #1254
 2019-04-16 21:22:48 -04:00
Rory O’Kane
6ffd434232 readme: mention --auto-hybrid-regex in advantages 
This feature solves a major reason I was skeptical of using ripgrep, so
I think it’s good to mention it in the section about why one should use
it.

I use backreferences a lot, so I had previously thought that ripgrep
would provide no speed advantage over ag, since I would always have
`-P` enabled. But when I saw `--auto-hybrid-regex` in the 11.0.0
changelog, I learned that ripgrep can use it to speed up simple queries
while still allowing me to write backreferences.

PR #1253
 2019-04-16 17:21:40 -04:00
Andrew Gallant
1f1cd9b467 pkg: update brew tap to 11.0.1 2019-04-16 13:39:56 -04:00
Andrew Gallant
973de50c9e ripgrep: release 11.0.1, take 2 2019-04-16 13:11:28 -04:00
Andrew Gallant
5f8805a496 ripgrep: release 11.0.1 2019-04-16 13:10:29 -04:00
Andrew Gallant
fdde2bcd38 deps: update regex to 1.1.6 
This brings in a fix for a regression introduced in ripgrep 11.

Fixes #1247
 2019-04-16 08:34:30 -04:00
Gerard de Melo
7b3fe6b325 doc: fix typo in FAQ 
PR #1248
 2019-04-16 08:32:30 -04:00
Max Horn
b3dd3ae203 ignore/types: add GAP 
Add support for file types used by the GAP language, a research system
computational discrete algebra, see <https://www.gap-system.org>

PR #1249
 2019-04-16 08:31:58 -04:00
Andrew Gallant
f3083e4574 readme: remove brew tap instructions 
The brew tap isn't really needed any more, since SIMD is now
automatically enabled in all binaries.
 2019-04-15 18:32:33 -04:00
Andrew Gallant
d03e30707e pkg: update brew tap to 11.0.0 2019-04-15 18:32:10 -04:00
Andrew Gallant
d7f57d9aab ripgrep: release 11.0.0 2019-04-15 18:09:40 -04:00
Andrew Gallant
1a2a24ea74 grep: release 0.2.4 2019-04-15 18:03:46 -04:00
Andrew Gallant
d66610b295 grep-cli: release 0.1.2 2019-04-15 18:02:44 -04:00
Andrew Gallant
019ae1989b grep-printer: release 0.1.2 2019-04-15 18:00:49 -04:00
Andrew Gallant
36d3f235dc grep-searcher: release 0.1.4 2019-04-15 17:59:22 -04:00
Andrew Gallant
79018eb693 grep-pcre2: release 0.1.3 2019-04-15 17:57:03 -04:00
Andrew Gallant
44cd344438 grep-regex: release 0.1.3 2019-04-15 17:56:04 -04:00
Andrew Gallant
e493e54b9b grep-matcher: release 0.1.2 2019-04-15 17:53:29 -04:00
Andrew Gallant
8e8215aa65 ignore: release 0.4.7 2019-04-15 17:50:37 -04:00
Andrew Gallant
3fe701498e doc: add note about --pre-glob 
There was a performance warning in the --pre docs, but didn't mention
--pre-glob as a possible mitigation to it.
 2019-04-15 17:47:48 -04:00
Andrew Gallant
e79085e9e4 release: globset 0.4.3 2019-04-15 14:07:03 -04:00
Andrew Gallant
764c197022 complete: fix typo 2019-04-15 07:04:57 -04:00
Andrew Gallant
ef1611b5f5 ripgrep: max-column-preview --> max-columns-preview 
Credit to @okdana for catching this. This naming is a bit more
consistent with the existing --max-columns flag.
 2019-04-15 06:51:51 -04:00
Andrew Gallant
45d12abbc5 changelog: small fixups 2019-04-14 20:21:55 -04:00
Andrew Gallant
5fde8391f9 changelog: backfill it 
I went through every commit since the 0.10.0 release and added anything
that I thought was missing.
 2019-04-14 20:04:01 -04:00
Marco Herrn
3edb11c513 ignore/types: add additional java files 
- .jspx for XHTML JSP files
- .properties for Java Properties files (resource bundles, etc.)

Closes #1242
 2019-04-14 19:38:24 -04:00
Andrew Gallant
ed144be775 ci: bump MSRV to 1.34.0 2019-04-14 19:29:27 -04:00
Andrew Gallant
967e7ad0de ripgrep: add --auto-hybrid-regex flag 
This flag, when set, will automatically dispatch to PCRE2 if the given
regex cannot be compiled by Rust's regex engine. If both engines fail to
compile the regex, then both errors are surfaced.

Closes #1155
 2019-04-14 19:29:27 -04:00
Andrew Gallant
9952ba2068 deps: update glob dev-dependency 2019-04-14 19:29:27 -04:00
Andrew Gallant
b751758d60 deps: update everything 2019-04-14 19:29:27 -04:00
Andrew Gallant
8f14cb18a5 ripgrep: increase pcre2's default JIT stack size 
The default stack size is 32KB, and this increases it to 10MB. 32KB is
pretty paltry in the environments in which ripgrep runs, and 10MB is
easily afforded as a maximum size. (The size limit we set for Rust's
regex engine is considerably larger.)

This was motivated due to the fack that JIT stack limits have been
observed to be hit in the wild:
https://github.com/Microsoft/vscode/issues/64606
 2019-04-14 19:29:27 -04:00
Andrew Gallant
da9d720431 ripgrep: add --pcre2-version flag 
This flag will output details about the version of PCRE2 that ripgrep
is using (if any).
 2019-04-14 19:29:27 -04:00
Andrew Gallant
a9d71a0368 pcre2: add a few re-exports 
This adds the top-level is_jit_available and version free functions from
the underlying pcre2 crate, and also forwards the max_jit_stack_size
option.
 2019-04-14 19:29:27 -04:00
Andrew Gallant
f3646242cc deps: use pcre2 0.2.0 
This comes with PCRE 10.32 and a few new options we'll use in subsequent
commits.
 2019-04-14 19:29:27 -04:00
Andrew Gallant
601f212a0b ripgrep: add -I as a short option for --no-filename 
This flag is commonly used in pipelines and it can be annoying to write
it out every time you need it.

Ideally, we would use -h for this to match GNU grep, but -h is used to
print help output.

Closes #1185
 2019-04-14 19:29:27 -04:00
Andrew Gallant
5a565354f8 versioning: next version will be ripgrep 11 
This sets up the release announcement and briefly describes the
versioning change. The actual version change itself won't happen until
the release.

Closes #1172
 2019-04-14 19:29:27 -04:00
Andrew Gallant
2a6532ae71 doc: note cases of exorbitant memory usage 
Fixes #1189
 2019-04-14 19:29:27 -04:00
Andrew Gallant
ece1f50cfe printer: support previews for long lines 
This commit adds support for showing a preview of long lines. While the
default still remains as completely suppressing the entire line, this
new functionality will show the first N graphemes of a matching line,
including the number of matches that are suppressed.

This was unfortunately a fairly invasive change to the printer that
required a bit of refactoring. On the bright side, the single line
and multi-line coloring are now more unified than they were before.

Closes #1078
 2019-04-14 19:29:27 -04:00
Andrew Gallant
a7d26c8f14 binary: rejigger ripgrep's handling of binary files 
This commit attempts to surface binary filtering in a slightly more
user friendly way. Namely, before, ripgrep would silently stop
searching a file if it detected a NUL byte, even if it had previously
printed a match. This can lead to the user quite reasonably assuming
that there are no more matches, since a partial search is fairly
unintuitive. (ripgrep has this behavior by default because it really
wants to NOT search binary files at all, just like it doesn't search
gitignored or hidden files.)

With this commit, if a match has already been printed and ripgrep detects
a NUL byte, then it will print a warning message indicating that the search
stopped prematurely.

Moreover, this commit adds a new flag, --binary, which causes ripgrep to
stop filtering binary files, but in a way that still avoids dumping
binary data into terminals. That is, the --binary flag makes ripgrep
behave more like grep's default behavior.

For files explicitly specified in a search, e.g., `rg foo some-file`,
then no binary filtering is applied (just like no gitignore and no
hidden file filtering is applied). Instead, ripgrep behaves as if you
gave the --binary flag for all explicitly given files.

This was a fairly invasive change, and potentially increases the UX
complexity of ripgrep around binary files. (Before, there were two
binary modes, where as now there are three.) However, ripgrep is now a
bit louder with warning messages when binary file detection might
otherwise be hiding potential matches, so hopefully this is a net
improvement.

Finally, the `-uuu` convenience now maps to `--no-ignore --hidden
--binary`, since this is closer to the actualy intent of the
`--unrestricted` flag, i.e., to reduce ripgrep's smart filtering. As a
consequence, `rg -uuu foo` should now search roughly the same number of
bytes as `grep -r foo`, and `rg -uuua foo` should search roughly the
same number of bytes as `grep -ra foo`. (The "roughly" weasel word is
used because grep's and ripgrep's binary file detection might differ
somewhat---perhaps based on buffer sizes---which can impact exactly what
is and isn't searched.)

See the numerous tests in tests/binary.rs for intended behavior.

Fixes #306, Fixes #855
 2019-04-14 19:29:27 -04:00
Andrew Gallant
bd222ae93f regex: fix HIR analysis bug 
An alternate can be empty at this point, so we must handle it. We didn't
before because the regex engine actually disallows empty alternates,
however, this code runs before the regex compiler rejects the regex.
 2019-04-14 19:29:27 -04:00
hupfdule
4359d8aac0 ignore/types: add more extensions for xml 
This includes:

    *.dtd for Document Type Definitions
    *.xsl and *.xslt for XSL Transformation descriptions
    *.xsd for XML Schema definitions
    *.xjb for JAXB bindings
    *.rng for Relax NG files
    *.sch for Schematron files

PR #1243
 2019-04-09 15:17:57 -04:00
tonypai
308819fb1f ignore/types: add lock files 
Treat anything with a `.lock` extension as a lock file, with
an extra rule or two for special cases, e.g., package-lock.json.
 2019-04-09 10:24:48 -04:00
Andrew Gallant
09108b7fda regex: make multi-literal searcher faster 
This makes the case of searching for a dictionary of a very large number
of literals much much faster. (~10x or so.) In particular, we achieve this
by short-circuiting the construction of a full regex when we know we have
a simple alternation of literals. Building the regex for a large dictionary
(>100,000 literals) turns out to be quite slow, even if it internally will
dispatch to Aho-Corasick.

Even that isn't quite enough. It turns out that even *parsing* such a regex
is quite slow. So when the -F/--fixed-strings flag is set, we short
circuit regex parsing completely and jump straight to Aho-Corasick.

We aren't quite as fast as GNU grep here, but it's much closer (less than
2x slower).

In general, this is somewhat of a hack. In particular, it seems plausible
that this optimization could be implemented entirely in the regex engine.
Unfortunately, the regex engine's internals are just not amenable to this
at all, so it would require a larger refactoring effort. For now, it's
good enough to add this fairly simple hack at a higher level.

Unfortunately, if you don't pass -F/--fixed-strings, then ripgrep will
be slower, because of the aforementioned missing optimization. Moreover,
passing flags like `-i` or `-S` will cause ripgrep to abandon this
optimization and fall back to something potentially much slower. Again,
this fix really needs to happen inside the regex engine, although we
might be able to special case -i when the input literals are pure ASCII
via Aho-Corasick's `ascii_case_insensitive`.

Fixes #497, Fixes #838
 2019-04-07 19:11:03 -04:00
Andrew Gallant
743d64f2e4 deps: update to clap 2.33 2019-04-06 10:35:08 -04:00
lesnyrumcajs
5962abc465 searcher: add option to disable BOM sniffing 
This commit adds a new encoding feature where the -E/--encoding flag
will now accept a value of 'none'. When given this value, all encoding
related machinery is disabled and ripgrep will search the raw bytes of
the file, including the BOM if it's present.

Closes #1207, Closes #1208
 2019-04-06 10:35:08 -04:00
dana
1604a18db3 ignore/types: add *.am and *.in for C/C++/make 
PR #1205
 2019-04-06 08:02:04 -04:00
luzpaz
9eeb0b01ce readme: add Repology badge 
This adds a badge to the README.md file indicating to users that click
on it if their os/distro carries that latest version of ripgrep.

PR #1213
 2019-04-06 08:00:40 -04:00
dana
df4400209a ripgrep: remove extra new-line after Clap output 
PR #1222
 2019-04-06 07:59:36 -04:00
Andrew Gallant
77439f99a4 deps: add bstr to Cargo.lock 2019-04-05 23:24:08 -04:00
Andrew Gallant
be7d6dd9ce regex: print out final regex in trace mode 
This is useful for debugging to see what regex is actually being run.
We put this as a trace since the regex can be quite gnarly. (It is not
pretty printed.)
 2019-04-05 23:24:08 -04:00
Andrew Gallant
9f15e3b671 regex: fix a perf bug when using -w flag 
When looking for an inner literal to speed up searches, if only a prefix
is found, then we generally give up doing inner literal optimizations since
the regex engine will generally handle it for us. Unfortunately, this
decision was being made *before* we wrap the regex in (^|\W)...($|\W) when
using the -w/--word-regexp flag, which would then defeat the literal
optimizations inside the regex engine.

We fix this with a bit of a hack that says, "if we're doing a word regexp,
then give me back any literal you find, even if it's a prefix."
 2019-04-05 23:24:08 -04:00
Andrew Gallant
254b8b67bb globset: small perf improvements 
This tweaks the path handling functions slightly to make them a hair
faster. In particular, `file_name` is called on every path that ripgrep
visits, and it was possible to remove a few branches without changing
behavior.
 2019-04-05 23:24:08 -04:00
Andrew Gallant
8a7f43b84d globset: use bstr 
This simplifies the various path related functions and pushed more platform
dependent code down into bstr. This likely also makes things a bit more
efficient on Windows, since we now only do a single UTF-8 check for each
file path.
 2019-04-05 23:24:08 -04:00
Andrew Gallant
d968a27ed5 cli: use bstr 
This uses bstr in the unescaping logic. This lets us remove some platform
specific code, and also lets us remove a hacked UTF-8 decoder on raw
bytes.
 2019-04-05 23:24:08 -04:00
Andrew Gallant
9b8f5cbaba config: switch to using bstrs 
This lets us implement correct Unicode trimming and also simplifies the
parsing logic a bit. This also removes the last platform specific bits of
code in ripgrep core.
 2019-04-05 23:24:08 -04:00
Andrew Gallant
c52da74ac3 printer: use bstr 
This starts the usage of bstr in the printer. We don't use it too much
yet, but it comes in handy for implementing PrinterPath and lets us push
down some platform specific code into bstr.
 2019-04-05 23:24:08 -04:00
Andrew Gallant
7dcbff9a9b searcher: partially migrate to bstr 
This commit causes grep-searcher to use byte strings internally for its
line buffer support. We manage to remove a use of `unsafe` by doing this
(by pushing it down into `bstr`).

We stop short of using byte strings everywhere else because we rely
heavily on the `impl ops::Index<[u8]> for grep_matcher::Match` impl,
which isn't available for byte strings. (It is premature to make bstr a
public dep of a core crate like grep-matcher, but maybe some day.)
 2019-04-05 23:24:08 -04:00
Andrew Gallant
bef1f0e770 ci: switch to xenial (#1234) 
Rust is having problems with trusty, in particular, see this bug I
filed: https://github.com/rust-lang/rust/issues/59411

This was purpotedly fixed in
https://github.com/rust-lang/rust/pull/59468,
but it appears the issue is still occurring.

This commit tries to update to Ubuntu 16.04 in the hope that it will fix
this problem.
 2019-04-03 19:52:34 -04:00
Andrew Gallant
cd9815cb37 deps: update to aho-corasick 0.7 
We do the simplest possible change to migrate to the new version.

Fixes #1228
 2019-04-03 13:51:26 -04:00
Andrew Gallant
3f22c3a658 deps: update everything 
This updates all dependencies to their latest versions.

We tolerate a duplicative aho-corasick for now, which we will fix in the
next commit.
 2019-04-03 13:07:26 -04:00
Andrew Gallant
0913972104 deps: bump encoding_rs_io 
This brings in a new API for disabling BOM sniffing.

This is part of the work toward completing
https://github.com/BurntSushi/ripgrep/issues/1207
 2019-03-03 16:36:34 -05:00
Andrew Gallant
f19b84fb23 regex: bump regex dep to fix match bug 
See

* 661bf53d5b
* edf45e6f5f

for details on the bug fix, which was in the regex engine.

Fixes #1203
 2019-02-27 17:42:14 -05:00
Andrew Gallant
59fc583aeb readme: include details about filtering 
Despite the fact that we mention this in several places, people are
still surprised by ripgrep's "smart" filtering.
 2019-02-27 08:01:23 -05:00
Andrew Gallant
1c7c4e6640 deps: update tempfile 2019-02-21 16:32:17 -05:00
Andrew Gallant
69c5e3938d deps: bump smallvec 
This gets rid of the unmaintained crates `unreachable` and `void`. Yay!
 2019-02-21 16:31:48 -05:00
Andrew Gallant
d9cf05ad50 deps: update to aho-corasick 0.6.10 
This brings in a fix for this bug:
https://github.com/BurntSushi/aho-corasick/issues/37

Fixes #1079
 2019-02-16 11:39:33 -05:00
Andrew Gallant
af8b6caebb deps: update various dependencies 2019-02-16 09:39:42 -05:00
Andrew Gallant
c84cfb6756 grep-regex-0.1.2 2019-02-16 09:30:06 -05:00
Andrew Gallant
895e26a000 ci: don't do releases on all tags 
This attempts to make Appveyor more conservative in what tags it thinks
are releases. I don't know for sure, but it looks like the previous
regex could match anywhere, so we anchor it.

Fixes #1195
 2019-02-10 12:51:56 -05:00
Andrew Gallant
8c95290ff6 deps: miscellaneous updates 2019-02-10 07:45:08 -05:00
Andrew Gallant
d6feeb7ff2 grep-searcher-0.1.3 2019-02-10 07:42:37 -05:00
Andrew Gallant
626ed00c19 searcher: revert big-endian patch 
This undoes the patch to stop using bytecount on big-endian
architectures. In particular, we bump our bytecount dependency to the
latest release, which has a fix.

This reverts commit a4868b8835.

Fixes #1144 (again), Closes #1194
 2019-02-10 07:40:32 -05:00
Andrew Gallant
332ad18401 tests: use const constructor for atomics 
We did this in 05411b2b for core ripgrep, but didn't carry it over to
tests.
 2019-02-09 16:27:25 -05:00
Andrew Gallant
fc3cf41247 grep-searcher-0.1.2 2019-02-09 16:13:07 -05:00
Andrew Gallant
a4868b8835 searcher: use naive line counting on big-endian 
This patches out bytecount's "fast" vectorized algorithm on big-endian
machines, where it has been observed to fail. Going forward, bytecount
should probably fix this on their end, but for now, we take a small
performance hit on big-endian machines.

Fixes #1144
 2019-02-09 16:13:07 -05:00
John Schmidt
f99b991117 ignore/types: add zig 
PR #1191
 2019-02-08 08:12:40 -05:00
Andrew Gallant
de0bc78982 deps: bump encoding_rs to 0.8.16 
This brings in an updated `encoding_rs` crate that uses `packed_simd`,
which compiles on the latest nightly. Compilation times do appear to be
impacted significantly though.

Fixes #1175 (again)
 2019-02-07 17:05:14 -05:00
Steffen Banhardt
147e96914c ignore/types: *.dtx and *.ins added for tex 
PR #1182
 2019-01-31 09:06:19 -05:00
Andrew Gallant
0abc40c23c readme: bump MSRV 
We bumped it a while back in the CI configuration, but didn't update the
README.
 2019-01-29 13:10:43 -05:00
Andrew Gallant
f768796e4f deps: update other deps 2019-01-29 13:08:56 -05:00
Andrew Gallant
da0c0c4705 deps: update to crossbeam-channel 0.3.8 
This drops dependencies on parking_lot and rand from ripgrep.

(rand is still used for tests.)
 2019-01-29 13:07:37 -05:00
Andrew Gallant
05411b2b32 deprecated: remove use of ATOMIC_BOOL_INIT 
Our MSRV is high enough that we can use const functions now.
 2019-01-29 13:05:16 -05:00
Andrew Gallant
cc93db3b18 cargo: include auto-generated message 
This is going to be annoying for a while if one switches between the
latest nightly compiler and older compilers. Sigh.
 2019-01-29 13:04:40 -05:00
Alex Macleod
049354b766 readme: remove EOL Fedora install instructions 
Fedora 27 and below are past their EOL, so it can now be said that it's
supported regularly on Fedora.

PR #1177
 2019-01-28 08:15:36 -05:00
Andrew Gallant
386dd2806d changelog: BUG #916 
This was fixed by bumping the MSRV above Rust 1.28.

Fixes #916
 2019-01-27 13:15:17 -05:00
Andrew Gallant
5fe9a954e6 changelog: BUG #1154 2019-01-27 13:05:50 -05:00
Andrew Gallant
f158a42a71 ignore: correctly detect hidden files on Windows 
This commit fixes a bug where ripgrep only treated files beginning with
a `.` as hidden. On Windows, we continue this tradition, but
additionally check whether a file has the special Windows "hidden"
attribute set. If so, we treat it as a hidden file.

In order to make this work without an additional stat call, we had to
rearrange some of the plumbing from the directory traverser.

Fixes #1154
 2019-01-27 12:11:52 -05:00
Andrew Gallant
5724391d39 doc: small updates to the FAQ and GUIDE 
Notably, ripgrep can do multiline search now. We also update the
supported compression format list and replace deprecated flags like
`--sort-files` with `--sort path`.
 2019-01-26 16:19:09 -05:00
Andrew Gallant
0df71240ff search: fix -F and -f interaction bug 
This fixes what appears to be a pretty egregious regression where the
`-F/--fixed-strings` flag wasn't be applied to patterns supplied via
the `-f/--file` flag. The same bug existed for the `-x/--line-regexp`
flag as well, which we fix here.

Fixes #1176
 2019-01-26 16:01:52 -05:00
Andrew Gallant
f3164f2615 exit: tweak exit status logic 
This changes how ripgrep emit exit status codes. In particular, any error
that occurs while searching will now cause ripgrep to emit a `2` exit
code, where as it previously would emit either a `0` or a `1` code based
on whether it matched or not. That is, ripgrep would only emit a `2` exit
code for a catastrophic error.

This tweak includes additional logic that GNU grep adheres to, which seems
like good sense. Namely, if -q/--quiet is given, and an error occurs and
a match occurs, then ripgrep will emit a `0` exit code.

Closes #1159
 2019-01-26 15:44:49 -05:00
Andrew Gallant
31d3e24130 args: prevent panicking in 'rg -h | rg' 
Previously, we relied on clap to handle printing either an error
message, or --help/--version output, in addition to setting the exit
status code. Unfortunately, for --help/--version output, clap was
panicking if the write failed, which can happen in fairly common
scenarios via a broken pipe error. e.g., `rg -h | head`.

We fix this by using clap's "safe" API and doing the printing ourselves.
We also set the exit code to `2` when an invalid command has been given.

Fixes #1125 and partially addresses #1159
 2019-01-26 14:39:40 -05:00
Andrew Gallant
bf842dbc7f doc: add note about inverted flags 
Fixes #1091
 2019-01-26 14:13:06 -05:00
Andrew Gallant
6d5dba85bd doc: clarify automatic encoding detection 
Fixes #1103
 2019-01-26 13:55:47 -05:00
Andrew Gallant
afb89bcdad fmt: shorten --ignore-file-case-insensitive description 2019-01-26 13:45:02 -05:00
Andrew Gallant
332dc56372 changelog: BUG #1095 2019-01-26 13:40:59 -05:00
Andrew Gallant
12a6ca45f9 config: add --no-ignore-dot flag 
This flag causes ripgrep to ignore `.ignore` files.

Closes #1138
 2019-01-26 13:40:12 -05:00
Andrew Gallant
9d703110cf regex: make CRLF hack more robust 
This commit improves the CRLF hack to be more robust. In particular, in
addition to rewriting `$` as `(?:\r??$)`, we now strip `\r` from the end
of a match if and only if the regex has an ending line anchor required for
a match. This doesn't quite make the hack 100% correct, but should fix most
use cases in practice. An example of a regex that will still be incorrect
is `foo|bar$`, since the analysis isn't quite sophisticated enough to
determine that a `\r` can be safely stripped from any match. Even if we
fix that, regexes like `foo\r|bar$` still won't be handled correctly. Alas,
more work on this front should really be focused on enabling this in the
regex engine itself.

The specific cause of this bug was that grep-searcher was sneakily
stripping CRLF from matching lines when it really shouldn't have. We remove
that code now, and instead rely on better match semantics provided at a
lower level.

Fixes #1095
 2019-01-26 12:34:28 -05:00
Andrew Gallant
e99b6bda0e deps: bump regex-syntax to 0.6.5 
This is necessary for the use of the new is_line_anchored_{start,end}
APIs.
 2019-01-26 12:20:02 -05:00
Andrew Gallant
276e2c9b9a searcher: always strip BOM 
This fixes a bug where a BOM prefix was included. While this was somewhat
intentional in order to have a faithful "UTF8 passthru" option, in
practice, this causes problems such as breaking patterns like `^` in a
really non-obvious way.

The actual fix was to add a new API to encoding_rs_io, which this commit
brings in.

Fixes #1163
 2019-01-25 17:18:57 -05:00
Andrew Gallant
9a9f54d44c readme: encoding_rs's SIMD support is broken 
Add a note about it to the README.

Also, remove mention of the avx-accel feature since it no longer exists.
(bytecount now uses runtime detection to enable SIMD support.)

Fixes #1175
 2019-01-24 07:00:53 -05:00
Andrew Gallant
47833b9ce7 deps: update removal of grep devdeps 2019-01-23 20:14:37 -05:00
Awad Mackie
44a9e37737 ignore/types: add method for retrieving file type definition 
Fixes #1116, Closes #1120
 2019-01-23 20:08:48 -05:00
Andrew Gallant
8fd05cacee changelog: BUG #1121 2019-01-23 20:06:01 -05:00
Rob Lourens
4691d11034 ripgrep: don't skip stdout in --files mode 
Specifically, this avoids triggering Windows antimalware when in --files mode.

See also #600.

Fixes #1121
 2019-01-23 20:04:44 -05:00
Andrew Gallant
519a6b68af grep: remove unused dependencies 
We remove these for now, but we'll eventually add them back once the
examples get more fleshed out.

Closes #1043
 2019-01-23 20:01:32 -05:00
Andrew Gallant
9c940b45f4 globset: permit ** to appear anywhere 
Previously, `man gitignore` specified that `**` was invalid unless it
was used in one of a few specific circumstances, i.e., `**`, `a/**`,
`**/b` or `a/**/b`. That is, `**` always had to be surrounded by either
a path separator or the beginning/end of the pattern.

It turns out that git itself has treated `**` outside the above contexts
as valid for quite a while, so there was an inconsistency between the
spec `man gitignore` and the implementation, and it wasn't clear which
was actually correct.

@okdana filed a bug against git[1] and got this fixed. The spec was wrong,
which has now been fixed [2] and updated[2].

This commit brings ripgrep in line with git and treats `**` outside of
the above contexts as two consecutive `*` patterns. We deprecate the
`InvalidRecursive` error since it is no longer used.

Fixes #373, Fixes #1098

[1] - https://public-inbox.org/git/C16A9F17-0375-42F9-90A9-A92C9F3D8BBA@dana.is
[2] - 627186d020
[3] - https://git-scm.com/docs/gitignore
 2019-01-23 19:59:39 -05:00
Andrew Gallant
0a167021c3 changelog: BUG #1174 2019-01-23 19:19:26 -05:00
Andrew Gallant
aeaa5fc1b1 globset: fix repeated use of ** 
This fixes a bug where repeated use of ** didn't behave as it should. In
particular, each use of `**` added a new requirement directory depth
requirement. For example, something like `**/**/b` would match
`foo/bar/b`, but it wouldn't match `foo/b` even though it should. In
particular, `**` semantics demand "infinite" depth, so repeated uses of
`**` should just coalesce as if only one was given.

We do this coalescing in the parser. It's a little tricky because we
treat `**/a`, `a/**` and `a/**/b` as distinct tokens with their own
regex conversions. We also test the crap out of it.

Fixes #1174
 2019-01-23 19:15:02 -05:00
Andrew Gallant
7048a06c31 changelog: BUG #1173 2019-01-23 18:14:16 -05:00
Andrew Gallant
23be3cf850 ignore: fix handling of ** 
When deciding whether to add the `**/` prefix or not, we should choose
not to add it if the pattern is simply a bare `**`. Previously, we were
only not adding it if it was `**/`, which is correct, but we also need
to do it for `**` since `**` can already match anywhere.

There's likely a more principled solution to this, but this works for
now.

Fixes #1173
 2019-01-23 18:12:35 -05:00
Andrew Gallant
b48bbf527d changelog: PR #1093 2019-01-23 17:56:18 -05:00
dana
8eabe47b57 ignore: always use literal_separator for gitignore patterns (#1093) 
PR #1093
 2019-01-23 17:54:28 -05:00
Michele Bologna
ff712bfd9d readme: add instructions for openSUSE 15.0 
PR #1088
 2019-01-22 21:46:11 -05:00
Mika Dede
a7f2d48234 printer: fix path handling in summarizer 
This commit fixes a bug where both of the following commands always
reported an error:

    rg --files-with-matches foo file
    rg --files-without-match foo file

In particular, the printer was erroneously respecting the `path` option
even the the summary kind was `PathWithMatch` or `PathWithoutMatch`. The
documented behavior is that those summary kinds always require a path,
and thus, the `path` option has no effect. We fix this by correcting the
case analysis.

This also fixes a bug where the exit code for `--files-without-match`
was not set correctly. We update the printer's `has_match` method to
report the correct value.

Fixes #1106, Closes #1130
 2019-01-22 21:37:23 -05:00
Andrew Gallant
57500ad013 changelog: brotli/zstd addition 2019-01-22 20:57:28 -05:00
dana
0b04553aff grep-cli: support Brotli/Zstd decompression 
Fixes #1099
 2019-01-22 20:56:16 -05:00
dana
1ae121122f ignore/types: add/update brotli, bzip2, gzip, xz, zstd 2019-01-22 20:56:16 -05:00
Andrew Gallant
688003e51c ripgrep: ban rustfmt 2019-01-22 20:07:26 -05:00
David Torosyan
718a00f6f2 ripgrep: add --ignore-file-case-insensitive 
The --ignore-file-case-insensitive flag causes all
.gitignore/.rgignore/.ignore files to have their globs matched without
regard for case. Because this introduces a potentially significant
performance regression, this is always disabled by default. Users that
need case insensitive matching can enable it on a case by case basis.

Closes #1164, Closes #1170
 2019-01-22 20:03:59 -05:00
Andrew Gallant
7cbc535d70 edition: fix build.rs 2019-01-19 10:46:57 -05:00
Andrew Gallant
7a6a40bae1 edition: move core ripgrep to Rust 2018 2019-01-19 10:44:30 -05:00
Andrew Gallant
1e9ee2cc85 deps: update memmap 2019-01-19 10:44:30 -05:00
Andrew Gallant
968491f8e9 deps: update to bytecount 0.5 
bytecount now uses runtime dispatch for enabling SIMD, which means we can
no longer need the avx-accel features. We remove it from ripgrep since the
next release will be a minor version bump, but leave them as no-ops for
the crates that previously used it.
 2019-01-19 10:44:30 -05:00
Andrew Gallant
63b0f31a22 deps: update various dependencies 
We also increase the MSRV to 1.32, the current stable release, which sets
the stage for migrating to Rust 2018.
 2019-01-19 10:44:30 -05:00
P M
7ecee299a5 ignore/types: add QML 
PR #1165
 2019-01-18 06:48:47 -05:00
David Håsäther
dd396ff34e doc: fix typo 
PR #1161
 2019-01-14 06:50:30 -05:00
Andrew Gallant
fb0a82f3c3 grep-printer: add macro docs, redux 2019-01-11 09:18:09 -05:00
Andrew Gallant
dbc8ca9cc1 grep-searcher: add docs for assert_eq_printed 
Looks like the deny(missing_docs) lint got a bit stronger.
 2019-01-11 09:03:00 -05:00
Marco Hinz
c3db8db93d doc: fix typo 2019-01-05 11:18:05 -05:00
Andrew Gallant
17ef4c40f3 ignore-0.4.6 2018-12-30 08:46:09 -05:00
Andrew Gallant
a9e0477ea8 ignore: permit use of deprecated trim_right 2018-12-30 08:44:59 -05:00
Andrew Gallant
b3c5773266 deps: bump ignore 2018-12-30 08:43:18 -05:00
Andrew Gallant
118b950085 ignore-0.4.5 2018-12-15 08:44:10 -05:00
Andrew Gallant
b45b2f58ea deps: update most other dependencies 
This commit is the result of doing:

  $ cargo update
  $ cargo update -p encoding_rs --precise 0.8.10

where the latter line prevents encoding_rs from updating to 0.8.11 (or
newer). In particular, the 0.8.11 release increased the minimum Rust
version to 1.29, where as ripgrep 0.10.x is still on 1.28. We stay on an
older version for now until ripgrep is ready to move to 0.11.x.
 2018-12-15 08:42:14 -05:00
Andrew Gallant
662a9bc73d deps: update to crossbeam-channel 0.3 
This also requires corresponding updates to both rand and rand_core. Doing
an update of rand without doing an update of rand_core results in
compilation errors because two distinct versions of rand_core are included
in the build, and the traits they expose are distinct and incompatible.

We also switch over to using tempfile instead of tempdir, which drops the
last remaining thing keeping rand 0.4 in the build.

Fixes #1141, Fixes #1142
 2018-12-15 08:40:04 -05:00
Andrew Gallant
401add0a99 deps: update regex and regex-syntax 
This brings in some new Unicode properties, such as \p{Emoji}.

It is now also technically possible construct a regex that recognizes
grapheme clusters.
 2018-12-09 16:33:37 -05:00
Simon Morgan
f81b72721b ignore/types: add ASP 
PR #1134
 2018-12-07 16:19:33 -05:00
Antony Lee
1d4fccaadc ignore/types: add postscript 
Although postscript/encapsulated postscript is usually thought of as a
binary format, it's actually mostly ASCII, so ripgrep will not ignore
these files.

The situation is basically the same as for pdf, which is also already
present in the list of known filetypes.

PR #1118
 2018-11-23 09:46:11 -05:00
Matteo Bertini
09e464e674 ignore/types: add more Cython file types 
From the [Cython file types](https://cython.readthedocs.io/en/latest/src/userguide/language_basics.html?highlight=pxi#cython-file-types) paragraph on the official docs:

> There are three file types in Cython:
>    The implementation files, carrying a .py or .pyx suffix.
>    The definition files, carrying a .pxd suffix.
>    The include files, carrying a .pxi suffix.

PR #1113
 2018-11-19 07:37:00 -05:00
Jon Parise
31adff6f3c ignore/types: add Apache Thrift 
PR #1102
 2018-11-07 07:42:13 -05:00
Andrew Gallant
b41e596327 doc: escape braces in AsciiDoc 
This commit fixes a bug where AsciiDoc would drop any line containing a
'{foo}' because it interpreted it as an undefined attribute reference:

> Simple attribute references take the form {<name>}. If the attribute name
> is defined its text value is substituted otherwise the line containing the
> reference is dropped from the output.

See: https://www.methods.co.nz/asciidoc/chunked/ch30.html

We fix this by simply replacing all occurrences of '{' and '}' with
their escaped forms: '&#123;' and '&#125;'.

Fixes #1101
 2018-11-06 06:57:16 -05:00
Andrew Gallant
fb62266620 deps: update encoding_rs 
This commit bumps the version of encoding_rs to use the latest release.
This appears to fix a panic in UTF-16 decoding.

Fixes #1089
 2018-10-22 06:50:35 -04:00
Dave Lee
acf226c39d ignore/types: add BUILD.bazel to bazel file type 
PR #1074
 2018-10-02 18:00:04 -04:00
Mathieu Bridon
8299625e48 ignore/types: add buildstream 
BuildStream is a Free Software tool for building/integrating software stacks.: https://buildstream.gitlab.io/buildstream/

It uses recipes written in YAML, in files with the `.bst` extension.

PR #1071
 2018-09-28 08:32:24 -04:00
Andrew Gallant
db256c87eb ripgrep: suggest -U/--multiline 
When a "\n literal is not allowed" error is reported, ripgrep will now
suggest the use of the -U/--multiline flag, which enables matching
newlines.

Fixes #1055
 2018-09-25 16:56:04 -04:00
Andrew Gallant
ba533f390e grep-searcher: update to encoding_rs_io 0.1.3 
This update includes a work-around for a presumed bug in encoding_rs
that causes a panic:
https://github.com/hsivonen/encoding_rs/issues/34

Specifically, to reproduce this in ripgrep, one can run the following:

    $ curl -LO https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.1.tar.gz
    $ tar xf ruby-2.5.1.tar.gz
    $ rg ZZZZZ ruby-2.5.1/test/rexml/data/t63-2.svg
    thread 'main' panicked at 'index out of bounds: the len is 1 but the index is 1'

Fixes #1052
 2018-09-25 16:56:04 -04:00
Andrew Gallant
ba503eb677 grep-regex: fix inner literal detection 
It seems the inner literal detector fails spectacularly in cases of
concatenations that involve groups. The issue here is that if the prefix
of a group inside a concatenation can match the empty string, then any
literals generated to that point in the concatenation need to be cut
such that they are never extended. The detector isn't really built to
handle this case, so we just act conservative cut literals whenever we
see a sub-group. This may make some regexes slower, but the inner
literal detector already misses plenty of cases.

Literal detection (including in the regex engine) is a key component
that needs to be completely rethought at some point.

Fixes #1064
 2018-09-25 16:56:04 -04:00
Andrew Gallant
f72c2dfd90 readme: touch up README 
Make the wording consistent.
 2018-09-14 11:33:56 -04:00
Sylvestre Ledru
c0aa58b4f7 Ripgrep is also available in Ubuntu (from Cosmic) 2018-09-14 08:41:05 +02:00
ykgmfq
184ee4c328 deb: add section info 
Put it in the same section as
https://packages.debian.org/stretch/grep

PR #1051
 2018-09-13 08:17:24 -04:00
Gabe Berke-Williams
e82fbf2c46 doc: fix typo 
"cretion" -> "creation"

PR #1045
 2018-09-10 06:49:48 -04:00
Andrew Gallant
eb18da0450 pcre2: use jit_if_available 
This will allow PCRE2 to fall back to non-JIT matching when running on
platforms without JIT support.

ref https://github.com/BurntSushi/rust-pcre2/issues/3
 2018-09-08 17:12:14 -04:00
Andrew Gallant
0f7494216f readme: update dpkg version 2018-09-08 10:46:40 -04:00
Andrew Chin
442a278635 readme: fancy regexes are not supported by default 
PR #1042
 2018-09-07 17:43:24 -04:00
Andrew Gallant
7ebed3ace6 pkg: update brew tap to 0.10.0 2018-09-07 14:43:59 -04:00
Andrew Gallant
8a7db1a918 ci: tweak deployment conditions 2018-09-07 14:07:52 -04:00
173 changed files with 10147 additions and 4949 deletions
Expand all files Collapse all files

26
ISSUE_TEMPLATE.md → .github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@@ -1,3 +1,12 @@
---
name: Bug report
about: An issue with ripgrep or any of its crates (ignore, globset, etc.)
title: ''
labels: ''
assignees: ''
---
#### What version of ripgrep are you using?
Replace this text with the output of `rg --version`.
@@ -12,18 +21,11 @@ Github binary releases.
Replace this text with your operating system and version.
#### Describe your question, feature request, or bug.
#### Describe your bug.
If a question, please describe the problem you're trying to solve and give
as much context as possible.
Give a high level description of the bug.
If a feature request, please describe the behavior you want and the motivation.
Please also provide an example of how ripgrep would be used if your feature
request were added.
If a bug, please see below.
#### If this is a bug, what are the steps to reproduce the behavior?
#### What are the steps to reproduce the behavior?
If possible, please include both your search patterns and the corpus on which
you are searching. Unless the bug is very obvious, then it is unlikely that it
@@ -32,7 +34,7 @@ will be fixed if the ripgrep maintainers cannot reproduce it.
If the corpus is too big and you cannot decrease its size, file the bug anyway
and the ripgrep maintainers will help figure out next steps.
#### If this is a bug, what is the actual behavior?
#### What is the actual behavior?
Show the command you ran and the actual output. Include the `--debug` flag in
your invocation of ripgrep.
@@ -48,6 +50,6 @@ goes
here
```
#### If this is a bug, what is the expected behavior?
#### What is the expected behavior?
What do you think ripgrep should have done?

6
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View File

@@ -0,0 +1,6 @@
blank_issues_enabled: true
contact_links:
- name: Ask a question
about: |
You've come to seek help or want to discuss something related to ripgrep.
url: https://github.com/BurntSushi/ripgrep/discussions/new

17
.github/ISSUE_TEMPLATE/feature_request.md vendored Normal file
View File

@@ -0,0 +1,17 @@
---
name: Feature request
about: Suggest a new feature for ripgrep
title: ''
labels: ''
assignees: ''
---
#### Describe your feature request
Please describe the behavior you want and the motivation. Please also provide
examples of how ripgrep would be used if your feature request were added.
If you're not sure what to write here, then try imagining what the ideal
documentation of your new feature would look like in ripgrep's man page. Then
try to write it.

198
.github/workflows/ci.yml vendored Normal file
View File

@@ -0,0 +1,198 @@
name: ci
on:
pull_request:
push:
branches:
- master
schedule:
- cron: '00 01 * * *'
jobs:
test:
name: test
env:
# For some builds, we use cross to test on 32-bit and big-endian
# systems.
CARGO: cargo
# When CARGO is set to CROSS, this is set to `--target matrix.target`.
TARGET_FLAGS:
# When CARGO is set to CROSS, TARGET_DIR includes matrix.target.
TARGET_DIR: ./target
# Emit backtraces on panics.
RUST_BACKTRACE: 1
runs-on: ${{ matrix.os }}
strategy:
matrix:
build:
# We test ripgrep on a pinned version of Rust, along with the moving
# targets of 'stable' and 'beta' for good measure.
- pinned
- stable
- beta
# Our release builds are generated by a nightly compiler to take
# advantage of the latest optimizations/compile time improvements. So
# we test all of them here. (We don't do mips releases, but test on
# mips for big-endian coverage.)
- nightly
- nightly-musl
- nightly-32
- nightly-mips
- nightly-arm
- macos
- win-msvc
- win-gnu
include:
- build: pinned
os: ubuntu-18.04
rust: 1.41.0
- build: stable
os: ubuntu-18.04
rust: stable
- build: beta
os: ubuntu-18.04
rust: beta
- build: nightly
os: ubuntu-18.04
rust: nightly
- build: nightly-musl
os: ubuntu-18.04
rust: nightly
target: x86_64-unknown-linux-musl
- build: nightly-32
os: ubuntu-18.04
rust: nightly
target: i686-unknown-linux-gnu
- build: nightly-mips
os: ubuntu-18.04
rust: nightly
target: mips64-unknown-linux-gnuabi64
- build: nightly-arm
os: ubuntu-18.04
rust: nightly
# For stripping release binaries:
# docker run --rm -v $PWD/target:/target:Z \
# rustembedded/cross:arm-unknown-linux-gnueabihf \
# arm-linux-gnueabihf-strip \
# /target/arm-unknown-linux-gnueabihf/debug/rg
target: arm-unknown-linux-gnueabihf
- build: macos
os: macos-latest
rust: nightly
- build: win-msvc
os: windows-2019
rust: nightly
- build: win-gnu
os: windows-2019
rust: nightly-x86_64-gnu
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Install packages (Ubuntu)
if: matrix.os == 'ubuntu-18.04'
run: |
ci/ubuntu-install-packages
- name: Install packages (macOS)
if: matrix.os == 'macos-latest'
run: |
ci/macos-install-packages
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: ${{ matrix.rust }}
profile: minimal
override: true
- name: Use Cross
if: matrix.target != ''
run: |
# FIXME: to work around bugs in latest cross release, install master.
# See: https://github.com/rust-embedded/cross/issues/357
cargo install --git https://github.com/rust-embedded/cross
echo "::set-env name=CARGO::cross"
echo "::set-env name=TARGET_FLAGS::--target ${{ matrix.target }}"
echo "::set-env name=TARGET_DIR::./target/${{ matrix.target }}"
- name: Show command used for Cargo
run: |
echo "cargo command is: ${{ env.CARGO }}"
echo "target flag is: ${{ env.TARGET_FLAGS }}"
- name: Build ripgrep and all crates
run: ${{ env.CARGO }} build --verbose --all ${{ env.TARGET_FLAGS }}
- name: Build ripgrep with PCRE2
run: ${{ env.CARGO }} build --verbose --all --features pcre2 ${{ env.TARGET_FLAGS }}
# This is useful for debugging problems when the expected build artifacts
# (like shell completions and man pages) aren't generated.
- name: Show build.rs stderr
shell: bash
run: |
set +x
stderr="$(find "${{ env.TARGET_DIR }}/debug" -name stderr -print0 | xargs -0 ls -t | head -n1)"
if [ -s "$stderr" ]; then
echo "===== $stderr ===== "
cat "$stderr"
echo "====="
fi
set -x
- name: Run tests with PCRE2 (sans cross)
if: matrix.target == ''
run: ${{ env.CARGO }} test --verbose --all --features pcre2 ${{ env.TARGET_FLAGS }}
- name: Run tests without PCRE2 (with cross)
# These tests should actually work, but they almost double the runtime.
# Every integration test spins up qemu to run 'rg', and when PCRE2 is
# enabled, every integration test is run twice: one with the default
# regex engine and once with PCRE2.
if: matrix.target != ''
run: ${{ env.CARGO }} test --verbose --all ${{ env.TARGET_FLAGS }}
- name: Test for existence of build artifacts (Windows)
if: matrix.os == 'windows-2019'
shell: bash
run: |
outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
ls "$outdir/_rg.ps1" && file "$outdir/_rg.ps1"
- name: Test for existence of build artifacts (Unix)
if: matrix.os != 'windows-2019'
shell: bash
run: |
outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
# TODO: Check for the man page generation here. For whatever reason,
# it seems to be intermittently failing in CI. No idea why.
# for f in rg.bash rg.fish rg.1; do
for f in rg.bash rg.fish; do
# We could use file -E here, but it isn't supported on macOS.
ls "$outdir/$f" && file "$outdir/$f"
done
- name: Test zsh shell completions (Unix, sans cross)
# We could test this when using Cross, but we'd have to execute the
# 'rg' binary (done in test-complete) with qemu, which is a pain and
# doesn't really gain us much. If shell completion works in one place,
# it probably works everywhere.
if: matrix.target == '' && matrix.os != 'windows-2019'
shell: bash
run: ci/test-complete
rustfmt:
name: rustfmt
runs-on: ubuntu-18.04
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
override: true
profile: minimal
components: rustfmt
- name: Check formatting
run: |
cargo fmt --all -- --check

211
.github/workflows/release.yml vendored Normal file
View File

@@ -0,0 +1,211 @@
# The way this works is a little weird. But basically, the create-release job
# runs purely to initialize the GitHub release itself. Once done, the upload
# URL of the release is saved as an artifact.
#
# The build-release job runs only once create-release is finished. It gets
# the release upload URL by downloading the corresponding artifact (which was
# uploaded by create-release). It then builds the release executables for each
# supported platform and attaches them as release assets to the previously
# created release.
#
# The key here is that we create the release only once.
name: release
on:
push:
# Enable when testing release infrastructure on a branch.
# branches:
# - ag/release
tags:
- '[0-9]+.[0-9]+.[0-9]+'
jobs:
create-release:
name: create-release
runs-on: ubuntu-latest
# env:
# Set to force version number, e.g., when no tag exists.
# RG_VERSION: TEST-0.0.0
steps:
- name: Create artifacts directory
run: mkdir artifacts
- name: Get the release version from the tag
if: env.RG_VERSION == ''
run: |
# Apparently, this is the right way to get a tag name. Really?
#
# See: https://github.community/t5/GitHub-Actions/How-to-get-just-the-tag-name/m-p/32167/highlight/true#M1027
echo "::set-env name=RG_VERSION::${GITHUB_REF#refs/tags/}"
echo "version is: ${{ env.RG_VERSION }}"
- name: Create GitHub release
id: release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ env.RG_VERSION }}
release_name: ${{ env.RG_VERSION }}
- name: Save release upload URL to artifact
run: echo "${{ steps.release.outputs.upload_url }}" > artifacts/release-upload-url
- name: Save version number to artifact
run: echo "${{ env.RG_VERSION }}" > artifacts/release-version
- name: Upload artifacts
uses: actions/upload-artifact@v1
with:
name: artifacts
path: artifacts
build-release:
name: build-release
needs: ['create-release']
runs-on: ${{ matrix.os }}
env:
# For some builds, we use cross to test on 32-bit and big-endian
# systems.
CARGO: cargo
# When CARGO is set to CROSS, this is set to `--target matrix.target`.
TARGET_FLAGS:
# When CARGO is set to CROSS, TARGET_DIR includes matrix.target.
TARGET_DIR: ./target
# Emit backtraces on panics.
RUST_BACKTRACE: 1
# Build static releases with PCRE2.
PCRE2_SYS_STATIC: 1
strategy:
matrix:
build: [linux, linux-arm, macos, win-msvc, win-gnu, win32-msvc]
include:
- build: linux
os: ubuntu-18.04
rust: nightly
target: x86_64-unknown-linux-musl
- build: linux-arm
os: ubuntu-18.04
rust: nightly
target: arm-unknown-linux-gnueabihf
- build: macos
os: macos-latest
rust: nightly
target: x86_64-apple-darwin
- build: win-msvc
os: windows-2019
rust: nightly
target: x86_64-pc-windows-msvc
- build: win-gnu
os: windows-2019
rust: nightly-x86_64-gnu
target: x86_64-pc-windows-gnu
- build: win32-msvc
os: windows-2019
rust: nightly
target: i686-pc-windows-msvc
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
fetch-depth: 1
- name: Install packages (Ubuntu)
if: matrix.os == 'ubuntu-18.04'
run: |
ci/ubuntu-install-packages
- name: Install packages (macOS)
if: matrix.os == 'macos-latest'
run: |
ci/macos-install-packages
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: ${{ matrix.rust }}
profile: minimal
override: true
target: ${{ matrix.target }}
- name: Use Cross
# if: matrix.os != 'windows-2019'
run: |
# FIXME: to work around bugs in latest cross release, install master.
# See: https://github.com/rust-embedded/cross/issues/357
cargo install --git https://github.com/rust-embedded/cross
echo "::set-env name=CARGO::cross"
echo "::set-env name=TARGET_FLAGS::--target ${{ matrix.target }}"
echo "::set-env name=TARGET_DIR::./target/${{ matrix.target }}"
- name: Show command used for Cargo
run: |
echo "cargo command is: ${{ env.CARGO }}"
echo "target flag is: ${{ env.TARGET_FLAGS }}"
echo "target dir is: ${{ env.TARGET_DIR }}"
- name: Get release download URL
uses: actions/download-artifact@v1
with:
name: artifacts
path: artifacts
- name: Set release upload URL and release version
shell: bash
run: |
release_upload_url="$(cat artifacts/release-upload-url)"
echo "::set-env name=RELEASE_UPLOAD_URL::$release_upload_url"
echo "release upload url: $RELEASE_UPLOAD_URL"
release_version="$(cat artifacts/release-version)"
echo "::set-env name=RELEASE_VERSION::$release_version"
echo "release version: $RELEASE_VERSION"
- name: Build release binary
run: ${{ env.CARGO }} build --verbose --release --features pcre2 ${{ env.TARGET_FLAGS }}
- name: Strip release binary (linux and macos)
if: matrix.build == 'linux' || matrix.build == 'macos'
run: strip "target/${{ matrix.target }}/release/rg"
- name: Strip release binary (arm)
if: matrix.build == 'linux-arm'
run: |
docker run --rm -v \
"$PWD/target:/target:Z" \
rustembedded/cross:arm-unknown-linux-gnueabihf \
arm-linux-gnueabihf-strip \
/target/arm-unknown-linux-gnueabihf/release/rg
- name: Build archive
shell: bash
run: |
outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
staging="ripgrep-${{ env.RELEASE_VERSION }}-${{ matrix.target }}"
mkdir -p "$staging"/{complete,doc}
cp {README.md,COPYING,UNLICENSE,LICENSE-MIT} "$staging/"
cp {CHANGELOG.md,FAQ.md,GUIDE.md} "$staging/doc/"
cp "$outdir"/{rg.bash,rg.fish,_rg.ps1} "$staging/complete/"
cp complete/_rg "$staging/complete/"
if [ "${{ matrix.os }}" = "windows-2019" ]; then
cp "target/${{ matrix.target }}/release/rg.exe" "$staging/"
7z a "$staging.zip" "$staging"
echo "::set-env name=ASSET::$staging.zip"
else
# The man page is only generated on Unix systems. ¯\_(ツ)_/¯
cp "$outdir"/rg.1 "$staging/doc/"
cp "target/${{ matrix.target }}/release/rg" "$staging/"
tar czf "$staging.tar.gz" "$staging"
echo "::set-env name=ASSET::$staging.tar.gz"
fi
- name: Upload release archive
uses: actions/upload-release-asset@v1.0.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ env.RELEASE_UPLOAD_URL }}
asset_path: ${{ env.ASSET }}
asset_name: ${{ env.ASSET }}
asset_content_type: application/octet-stream

108
.travis.yml
View File

@@ -1,108 +0,0 @@
language: rust
env:
global:
- PROJECT_NAME: ripgrep
- RUST_BACKTRACE: full
addons:
apt:
packages:
# For generating man page.
- libxslt1-dev
- asciidoc
- docbook-xsl
- xsltproc
- libxml2-utils
# Needed for completion-function test.
- zsh
# Needed for testing decompression search.
- xz-utils
- liblz4-tool
# For building MUSL static builds on Linux.
- musl-tools
matrix:
fast_finish: true
include:
# Nightly channel.
# All *nix releases are done on the nightly channel to take advantage
# of the regex library's multiple pattern SIMD search.
- os: linux
rust: nightly
env: TARGET=i686-unknown-linux-musl
- os: linux
rust: nightly
env: TARGET=x86_64-unknown-linux-musl
- os: osx
rust: nightly
# XML_CATALOG_FILES is apparently necessary for asciidoc on macOS.
env: TARGET=x86_64-apple-darwin XML_CATALOG_FILES=/usr/local/etc/xml/catalog
- os: linux
rust: nightly
env: TARGET=arm-unknown-linux-gnueabihf GCC_VERSION=4.8
addons:
apt:
packages:
- gcc-4.8-arm-linux-gnueabihf
- binutils-arm-linux-gnueabihf
- libc6-armhf-cross
- libc6-dev-armhf-cross
# For generating man page.
- libxslt1-dev
- asciidoc
- docbook-xsl
- xsltproc
- libxml2-utils
# Beta channel. We enable these to make sure there are no regressions in
# Rust beta releases.
- os: linux
rust: beta
env: TARGET=x86_64-unknown-linux-musl
- os: linux
rust: beta
env: TARGET=x86_64-unknown-linux-gnu
# Minimum Rust supported channel. We enable these to make sure ripgrep
# continues to work on the advertised minimum Rust version.
- os: linux
rust: 1.28.0
env: TARGET=x86_64-unknown-linux-gnu
- os: linux
rust: 1.28.0
env: TARGET=x86_64-unknown-linux-musl
- os: linux
rust: 1.28.0
env: TARGET=arm-unknown-linux-gnueabihf GCC_VERSION=4.8
addons:
apt:
packages:
- gcc-4.8-arm-linux-gnueabihf
- binutils-arm-linux-gnueabihf
- libc6-armhf-cross
- libc6-dev-armhf-cross
# For generating man page.
- libxslt1-dev
- asciidoc
- docbook-xsl
- xsltproc
- libxml2-utils
install: ci/install.sh
script: ci/script.sh
before_deploy: ci/before_deploy.sh
deploy:
provider: releases
file_glob: true
file: deployment/${PROJECT_NAME}-${TRAVIS_TAG}-${TARGET}.tar.gz
skip_cleanup: true
on:
condition: $TRAVIS_RUST_VERSION = nightly
tags: true
api_key:
secure: "IbSnsbGkxSydR/sozOf1/SRvHplzwRUHzcTjM7BKnr7GccL86gRPUrsrvD103KjQUGWIc1TnK1YTq5M0Onswg/ORDjqa1JEJPkPdPnVh9ipbF7M2De/7IlB4X4qXLKoApn8+bx2x/mfYXu4G+G1/2QdbaKK2yfXZKyjz0YFx+6CNrVCT2Nk8q7aHvOOzAL58vsG8iPDpupuhxlMDDn/UhyOWVInmPPQ0iJR1ZUJN8xJwXvKvBbfp3AhaBiAzkhXHNLgBR8QC5noWWMXnuVDMY3k4f3ic0V+p/qGUCN/nhptuceLxKFicMCYObSZeUzE5RAI0/OBW7l3z2iCoc+TbAnn+JrX/ObJCfzgAOXAU3tLaBFMiqQPGFKjKg1ltSYXomOFP/F7zALjpvFp4lYTBajRR+O3dqaxA9UQuRjw27vOeUpMcga4ZzL4VXFHzrxZKBHN//XIGjYAVhJ1NSSeGpeJV5/+jYzzWKfwSagRxQyVCzMooYFFXzn8Yxdm3PJlmp3GaAogNkdB9qKcrEvRINCelalzALPi0hD/HUDi8DD2PNTCLLMo6VSYtvc685Zbe+KgNzDV1YyTrRCUW6JotrS0r2ULLwnsh40hSB//nNv3XmwNmC/CmW5QAnIGj8cBMF4S2t6ohADIndojdAfNiptmaZOIT6owK7bWMgPMyopo="
branches:
only:
# Pushes and PR to the master branch
- master
# Ruby regex to match tags. Required, or travis won't trigger deploys when
# a new tag is pushed.
- /^\d+\.\d+\.\d+.*$/
notifications:
email:
on_success: never

327
CHANGELOG.md
View File

@@ -1,3 +1,330 @@
12.1.1 (2020-05-29)
===================
ripgrep 12.1.1 is a patch release that fixes a couple small bugs. In
particular, the ripgrep 12.1.0 release did not tag new releases for all of its
in-tree dependencies. As a result, ripgrep built dependencies from crates.io
would produce a different build than compiling ripgrep from source on the
`12.1.0` tag. Namely, some crates like `grep-cli` had unreleased changes.
Bug fixes:
* [BUG #1581](https://github.com/BurntSushi/ripgrep/issues/1581):
Corrects some egregious markup output in `--help`.
* [BUG #1591](https://github.com/BurntSushi/ripgrep/issues/1591):
Mention the special `$0` capture group in docs for the `-r/--replace` flag.
* [BUG #1602](https://github.com/BurntSushi/ripgrep/issues/1602):
Fix failing test resulting from out-of-sync dependencies.
12.1.0 (2020-05-09)
===================
ripgrep 12.1.0 is a small minor version release that mostly includes bug fixes
and documentation improvements. This release also contains some important
notices for downstream packagers.
**Notices for downstream ripgrep package maintainers:**
* Fish shell completions will be removed in the ripgrep 13 release.
See [#1577](https://github.com/BurntSushi/ripgrep/issues/1577)
for more details.
* ripgrep has switched from `a2x` to `asciidoctor` to generate the man page.
If `asciidoctor` is not present, then ripgrep will currently fall back to
`a2x`. Support for `a2x` will be dropped in the ripgrep 13 release.
See [#1544](https://github.com/BurntSushi/ripgrep/issues/1544)
for more details.
Feature enhancements:
* [FEATURE #1547](https://github.com/BurntSushi/ripgrep/pull/1547):
Support decompressing `.Z` files via `uncompress`.
Bug fixes:
* [BUG #1252](https://github.com/BurntSushi/ripgrep/issues/1252):
Add a section on the `--pre` flag to the GUIDE.
* [BUG #1339](https://github.com/BurntSushi/ripgrep/issues/1339):
Improve error message when a pattern with invalid UTF-8 is provided.
* [BUG #1524](https://github.com/BurntSushi/ripgrep/issues/1524):
Note how to escape a `$` when using `--replace`.
* [BUG #1537](https://github.com/BurntSushi/ripgrep/issues/1537):
Fix match bug caused by inner literal optimization.
* [BUG #1544](https://github.com/BurntSushi/ripgrep/issues/1544):
ripgrep now uses `asciidoctor` instead of `a2x` to generate its man page.
* [BUG #1550](https://github.com/BurntSushi/ripgrep/issues/1550):
Substantially reduce peak memory usage when searching wide directories.
* [BUG #1571](https://github.com/BurntSushi/ripgrep/issues/1571):
Add note about configuration files in `--type-{add,clear}` docs.
* [BUG #1573](https://github.com/BurntSushi/ripgrep/issues/1573):
Fix incorrect `--count-matches` output when using look-around.
12.0.1 (2020-03-29)
===================
ripgrep 12.0.1 is a small patch release that includes a minor bug fix relating
to superfluous error messages when searching git repositories with sub-modules.
This was a regression introduced in the 12.0.0 release.
Bug fixes:
* [BUG #1520](https://github.com/BurntSushi/ripgrep/issues/1520):
Don't emit spurious error messages in git repositories with submodules.
12.0.0 (2020-03-15)
===================
ripgrep 12 is a new major version release of ripgrep that contains many bug
fixes, several important performance improvements and a few minor new features.
In a near future release, I am hoping to add an
[indexing feature](https://github.com/BurntSushi/ripgrep/issues/1497)
to ripgrep, which will dramatically speed up searching by building an index.
Feedback would very much be appreciated, especially on the user experience
which will be difficult to get right.
This release has no known breaking changes.
Deprecations:
* The `--no-pcre2-unicode` flag is deprecated. Instead, use the `--no-unicode`
flag, which applies to both the default regex engine and PCRE2. For now,
`--no-pcre2-unicode` and `--pcre2-unicode` are aliases to `--no-unicode`
and `--unicode`, respectively. The `--[no-]pcre2-unicode` flags may be
removed in a future release.
* The `--auto-hybrid-regex` flag is deprecated. Instead, use the new `--engine`
flag with the `auto` value.
Performance improvements:
* [PERF #1087](https://github.com/BurntSushi/ripgrep/pull/1087):
ripgrep is smarter when detected literals are whitespace.
* [PERF #1381](https://github.com/BurntSushi/ripgrep/pull/1381):
Directory traversal is sped up with speculative ignore-file existence checks.
* [PERF cd8ec38a](https://github.com/BurntSushi/ripgrep/commit/cd8ec38a):
Improve inner literal detection to cover more cases more effectively.
e.g., ` +Sherlock Holmes +` now has ` Sherlock Holmes ` extracted instead
of ` `.
* [PERF 6a0e0147](https://github.com/BurntSushi/ripgrep/commit/6a0e0147):
Improve literal detection when the `-w/--word-regexp` flag is used.
* [PERF ad97e9c9](https://github.com/BurntSushi/ripgrep/commit/ad97e9c9):
Improve overall performance of the `-w/--word-regexp` flag.
Feature enhancements:
* Added or improved file type filtering for erb, diff, Gradle, HAML, Org,
Postscript, Skim, Slim, Slime, RPM Spec files, Typoscript, xml.
* [FEATURE #1370](https://github.com/BurntSushi/ripgrep/pull/1370):
Add `--include-zero` flag that shows files searched without matches.
* [FEATURE #1390](https://github.com/BurntSushi/ripgrep/pull/1390):
Add `--no-context-separator` flag that always hides context separators.
* [FEATURE #1414](https://github.com/BurntSushi/ripgrep/pull/1414):
Add `--no-require-git` flag to allow ripgrep to respect gitignores anywhere.
* [FEATURE #1420](https://github.com/BurntSushi/ripgrep/pull/1420):
Add `--no-ignore-exclude` to disregard rules in `.git/info/exclude` files.
* [FEATURE #1466](https://github.com/BurntSushi/ripgrep/pull/1466):
Add `--no-ignore-files` flag to disable all `--ignore-file` flags.
* [FEATURE #1488](https://github.com/BurntSushi/ripgrep/pull/1488):
Add '--engine' flag for easier switching between regex engines.
* [FEATURE 75cbe88f](https://github.com/BurntSushi/ripgrep/commit/75cbe88f):
Add `--no-unicode` flag. This works on all supported regex engines.
Bug fixes:
* [BUG #1291](https://github.com/BurntSushi/ripgrep/issues/1291):
ripgrep now works in non-existent directories.
* [BUG #1319](https://github.com/BurntSushi/ripgrep/issues/1319):
Fix match bug due to errant literal detection.
* [**BUG #1335**](https://github.com/BurntSushi/ripgrep/issues/1335):
Fixes a performance bug when searching plain text files with very long lines.
This was a serious performance regression in some cases.
* [BUG #1344](https://github.com/BurntSushi/ripgrep/issues/1344):
Document usage of `--type all`.
* [BUG #1389](https://github.com/BurntSushi/ripgrep/issues/1389):
Fixes a bug where ripgrep would panic when searching a symlinked directory.
* [BUG #1439](https://github.com/BurntSushi/ripgrep/issues/1439):
Improve documentation for ripgrep's automatic stdin detection.
* [BUG #1441](https://github.com/BurntSushi/ripgrep/issues/1441):
Remove CPU features from man page.
* [BUG #1442](https://github.com/BurntSushi/ripgrep/issues/1442),
[BUG #1478](https://github.com/BurntSushi/ripgrep/issues/1478):
Improve documentation of the `-g/--glob` flag.
* [BUG #1445](https://github.com/BurntSushi/ripgrep/issues/1445):
ripgrep now respects ignore rules from .git/info/exclude in worktrees.
* [BUG #1485](https://github.com/BurntSushi/ripgrep/issues/1485):
Fish shell completions from the release Debian package are now installed to
`/usr/share/fish/vendor_completions.d/rg.fish`.
11.0.2 (2019-08-01)
===================
ripgrep 11.0.2 is a new patch release that fixes a few bugs, including a
performance regression and a matching bug when using the `-F/--fixed-strings`
flag.
Feature enhancements:
* [FEATURE #1293](https://github.com/BurntSushi/ripgrep/issues/1293):
Added `--glob-case-insensitive` flag that makes `--glob` behave as `--iglob`.
Bug fixes:
* [BUG #1246](https://github.com/BurntSushi/ripgrep/issues/1246):
Add translations to README, starting with an unofficial Chinese translation.
* [BUG #1259](https://github.com/BurntSushi/ripgrep/issues/1259):
Fix bug where the last byte of a `-f file` was stripped if it wasn't a `\n`.
* [BUG #1261](https://github.com/BurntSushi/ripgrep/issues/1261):
Document that no error is reported when searching for `\n` with `-P/--pcre2`.
* [BUG #1284](https://github.com/BurntSushi/ripgrep/issues/1284):
Mention `.ignore` and `.rgignore` more prominently in the README.
* [BUG #1292](https://github.com/BurntSushi/ripgrep/issues/1292):
Fix bug where `--with-filename` was sometimes enabled incorrectly.
* [BUG #1268](https://github.com/BurntSushi/ripgrep/issues/1268):
Fix major performance regression in GitHub `x86_64-linux` binary release.
* [BUG #1302](https://github.com/BurntSushi/ripgrep/issues/1302):
Show better error messages when a non-existent preprocessor command is given.
* [BUG #1334](https://github.com/BurntSushi/ripgrep/issues/1334):
Fix match regression with `-F` flag when patterns contain meta characters.
11.0.1 (2019-04-16)
===================
ripgrep 11.0.1 is a new patch release that fixes a search regression introduced
in the previous 11.0.0 release. In particular, ripgrep can enter an infinite
loop for some search patterns when searching invalid UTF-8.
Bug fixes:
* [BUG #1247](https://github.com/BurntSushi/ripgrep/issues/1247):
Fix search bug that can cause ripgrep to enter an infinite loop.
11.0.0 (2019-04-15)
===================
ripgrep 11 is a new major version release of ripgrep that contains many bug
fixes, some performance improvements and a few feature enhancements. Notably,
ripgrep's user experience for binary file filtering has been improved. See the
[guide's new section on binary data](GUIDE.md#binary-data) for more details.
This release also marks a change in ripgrep's versioning. Where as the previous
version was `0.10.0`, this version is `11.0.0`. Moving forward, ripgrep's
major version will be increased a few times per year. ripgrep will continue to
be conservative with respect to backwards compatibility, but may occasionally
introduce breaking changes, which will always be documented in this CHANGELOG.
See [issue 1172](https://github.com/BurntSushi/ripgrep/issues/1172) for a bit
more detail on why this versioning change was made.
This release increases the **minimum supported Rust version** from 1.28.0 to
1.34.0.
**BREAKING CHANGES**:
* ripgrep has tweaked its exit status codes to be more like GNU grep's. Namely,
if a non-fatal error occurs during a search, then ripgrep will now always
emit a `2` exit status code, regardless of whether a match is found or not.
Previously, ripgrep would only emit a `2` exit status code for a catastrophic
error (e.g., regex syntax error). One exception to this is if ripgrep is run
with `-q/--quiet`. In that case, if an error occurs and a match is found,
then ripgrep will exit with a `0` exit status code.
* Supplying the `-u/--unrestricted` flag three times is now equivalent to
supplying `--no-ignore --hidden --binary`. Previously, `-uuu` was equivalent
to `--no-ignore --hidden --text`. The difference is that `--binary` disables
binary file filtering without potentially dumping binary data into your
terminal. That is, `rg -uuu foo` should now be equivalent to `grep -r foo`.
* The `avx-accel` feature of ripgrep has been removed since it is no longer
necessary. All uses of AVX in ripgrep are now enabled automatically via
runtime CPU feature detection. The `simd-accel` feature does remain available
(only for enabling SIMD for transcoding), however, it does increase
compilation times substantially at the moment.
Performance improvements:
* [PERF #497](https://github.com/BurntSushi/ripgrep/issues/497),
[PERF #838](https://github.com/BurntSushi/ripgrep/issues/838):
Make `rg -F -f dictionary-of-literals` much faster.
Feature enhancements:
* Added or improved file type filtering for Apache Thrift, ASP, Bazel, Brotli,
BuildStream, bzip2, C, C++, Cython, gzip, Java, Make, Postscript, QML, Tex,
XML, xz, zig and zstd.
* [FEATURE #855](https://github.com/BurntSushi/ripgrep/issues/855):
Add `--binary` flag for disabling binary file filtering.
* [FEATURE #1078](https://github.com/BurntSushi/ripgrep/pull/1078):
Add `--max-columns-preview` flag for showing a preview of long lines.
* [FEATURE #1099](https://github.com/BurntSushi/ripgrep/pull/1099):
Add support for Brotli and Zstd to the `-z/--search-zip` flag.
* [FEATURE #1138](https://github.com/BurntSushi/ripgrep/pull/1138):
Add `--no-ignore-dot` flag for ignoring `.ignore` files.
* [FEATURE #1155](https://github.com/BurntSushi/ripgrep/pull/1155):
Add `--auto-hybrid-regex` flag for automatically falling back to PCRE2.
* [FEATURE #1159](https://github.com/BurntSushi/ripgrep/pull/1159):
ripgrep's exit status logic should now match GNU grep. See updated man page.
* [FEATURE #1164](https://github.com/BurntSushi/ripgrep/pull/1164):
Add `--ignore-file-case-insensitive` for case insensitive ignore globs.
* [FEATURE #1185](https://github.com/BurntSushi/ripgrep/pull/1185):
Add `-I` flag as a short option for the `--no-filename` flag.
* [FEATURE #1207](https://github.com/BurntSushi/ripgrep/pull/1207):
Add `none` value to `-E/--encoding` to forcefully disable all transcoding.
* [FEATURE da9d7204](https://github.com/BurntSushi/ripgrep/commit/da9d7204):
Add `--pcre2-version` for querying showing PCRE2 version information.
Bug fixes:
* [BUG #306](https://github.com/BurntSushi/ripgrep/issues/306),
[BUG #855](https://github.com/BurntSushi/ripgrep/issues/855):
Improve the user experience for ripgrep's binary file filtering.
* [BUG #373](https://github.com/BurntSushi/ripgrep/issues/373),
[BUG #1098](https://github.com/BurntSushi/ripgrep/issues/1098):
`**` is now accepted as valid syntax anywhere in a glob.
* [BUG #916](https://github.com/BurntSushi/ripgrep/issues/916):
ripgrep no longer hangs when searching `/proc` with a zombie process present.
* [BUG #1052](https://github.com/BurntSushi/ripgrep/issues/1052):
Fix bug where ripgrep could panic when transcoding UTF-16 files.
* [BUG #1055](https://github.com/BurntSushi/ripgrep/issues/1055):
Suggest `-U/--multiline` when a pattern contains a `\n`.
* [BUG #1063](https://github.com/BurntSushi/ripgrep/issues/1063):
Always strip a BOM if it's present, even for UTF-8.
* [BUG #1064](https://github.com/BurntSushi/ripgrep/issues/1064):
Fix inner literal detection that could lead to incorrect matches.
* [BUG #1079](https://github.com/BurntSushi/ripgrep/issues/1079):
Fixes a bug where the order of globs could result in missing a match.
* [BUG #1089](https://github.com/BurntSushi/ripgrep/issues/1089):
Fix another bug where ripgrep could panic when transcoding UTF-16 files.
* [BUG #1091](https://github.com/BurntSushi/ripgrep/issues/1091):
Add note about inverted flags to the man page.
* [BUG #1093](https://github.com/BurntSushi/ripgrep/pull/1093):
Fix handling of literal slashes in gitignore patterns.
* [BUG #1095](https://github.com/BurntSushi/ripgrep/issues/1095):
Fix corner cases involving the `--crlf` flag.
* [BUG #1101](https://github.com/BurntSushi/ripgrep/issues/1101):
Fix AsciiDoc escaping for man page output.
* [BUG #1103](https://github.com/BurntSushi/ripgrep/issues/1103):
Clarify what `--encoding auto` does.
* [BUG #1106](https://github.com/BurntSushi/ripgrep/issues/1106):
`--files-with-matches` and `--files-without-match` work with one file.
* [BUG #1121](https://github.com/BurntSushi/ripgrep/issues/1121):
Fix bug that was triggering Windows antimalware when using the `--files`
flag.
* [BUG #1125](https://github.com/BurntSushi/ripgrep/issues/1125),
[BUG #1159](https://github.com/BurntSushi/ripgrep/issues/1159):
ripgrep shouldn't panic for `rg -h | rg` and should emit correct exit status.
* [BUG #1144](https://github.com/BurntSushi/ripgrep/issues/1144):
Fixes a bug where line numbers could be wrong on big-endian machines.
* [BUG #1154](https://github.com/BurntSushi/ripgrep/issues/1154):
Windows files with "hidden" attribute are now treated as hidden.
* [BUG #1173](https://github.com/BurntSushi/ripgrep/issues/1173):
Fix handling of `**` patterns in gitignore files.
* [BUG #1174](https://github.com/BurntSushi/ripgrep/issues/1174):
Fix handling of repeated `**` patterns in gitignore files.
* [BUG #1176](https://github.com/BurntSushi/ripgrep/issues/1176):
Fix bug where `-F`/`-x` weren't applied to patterns given via `-f`.
* [BUG #1189](https://github.com/BurntSushi/ripgrep/issues/1189):
Document cases where ripgrep may use a lot of memory.
* [BUG #1203](https://github.com/BurntSushi/ripgrep/issues/1203):
Fix a matching bug related to the suffix literal optimization.
* [BUG 8f14cb18](https://github.com/BurntSushi/ripgrep/commit/8f14cb18):
Increase the default stack size for PCRE2's JIT.
0.10.0 (2018-09-07)
===================
This is a new minor version release of ripgrep that contains some major new

782
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

52
Cargo.toml
View File

@@ -1,11 +1,11 @@
[package]
name = "ripgrep"
version = "0.10.0" #:version
version = "12.1.1" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """
ripgrep is a line-oriented search tool that recursively searches your current
directory for a regex pattern while respecting your gitignore rules. ripgrep
has first class support on Windows, macOS and Linux
has first class support on Windows, macOS and Linux.
"""
documentation = "https://github.com/BurntSushi/ripgrep"
homepage = "https://github.com/BurntSushi/ripgrep"
@@ -17,14 +17,11 @@ license = "Unlicense OR MIT"
exclude = ["HomebrewFormula"]
build = "build.rs"
autotests = false
[badges]
travis-ci = { repository = "BurntSushi/ripgrep" }
appveyor = { repository = "BurntSushi/ripgrep" }
edition = "2018"
[[bin]]
bench = false
path = "src/main.rs"
path = "crates/core/main.rs"
name = "rg"
[[test]]
@@ -33,46 +30,50 @@ path = "tests/tests.rs"
[workspace]
members = [
"globset",
"grep",
"grep-cli",
"grep-matcher",
"grep-pcre2",
"grep-printer",
"grep-regex",
"grep-searcher",
"ignore",
"crates/globset",
"crates/grep",
"crates/cli",
"crates/matcher",
"crates/pcre2",
"crates/printer",
"crates/regex",
"crates/searcher",
"crates/ignore",
]
[dependencies]
grep = { version = "0.2.2", path = "grep" }
ignore = { version = "0.4.4", path = "ignore" }
bstr = "0.2.12"
grep = { version = "0.2.7", path = "crates/grep" }
ignore = { version = "0.4.16", path = "crates/ignore" }
lazy_static = "1.1.0"
log = "0.4.5"
num_cpus = "1.8.0"
regex = "1.0.5"
regex = "1.3.5"
serde_json = "1.0.23"
termcolor = "1.0.3"
termcolor = "1.1.0"
[dependencies.clap]
version = "2.32.0"
version = "2.33.0"
default-features = false
features = ["suggestions"]
[target.'cfg(all(target_env = "musl", target_pointer_width = "64"))'.dependencies.jemallocator]
version = "0.3.0"
[build-dependencies]
lazy_static = "1.1.0"
[build-dependencies.clap]
version = "2.32.0"
version = "2.33.0"
default-features = false
features = ["suggestions"]
[dev-dependencies]
serde = "1.0.77"
serde_derive = "1.0.77"
walkdir = "2"
[features]
avx-accel = ["grep/avx-accel"]
simd-accel = ["grep/simd-accel"]
pcre2 = ["grep/pcre2"]
@@ -81,6 +82,7 @@ debug = 1
[package.metadata.deb]
features = ["pcre2"]
section = "utils"
assets = [
["target/release/rg", "usr/bin/", "755"],
["COPYING", "usr/share/doc/ripgrep/", "644"],
@@ -92,11 +94,11 @@ assets = [
# The man page is automatically generated by ripgrep's build process, so
# this file isn't actually commited. Instead, to create a dpkg, either
# create a deployment/deb directory and copy the man page to it, or use the
# 'ci/build_deb.sh' script.
# 'ci/build-deb' script.
["deployment/deb/rg.1", "usr/share/man/man1/rg.1", "644"],
# Similarly for shell completions.
["deployment/deb/rg.bash", "usr/share/bash-completion/completions/rg", "644"],
["deployment/deb/rg.fish", "usr/share/fish/completions/rg.fish", "644"],
["deployment/deb/rg.fish", "usr/share/fish/vendor_completions.d/rg.fish", "644"],
["deployment/deb/_rg", "usr/share/zsh/vendor-completions/", "644"],
]
extended-description = """\

11
Cross.toml Normal file
View File

@@ -0,0 +1,11 @@
[target.x86_64-unknown-linux-musl]
image = "burntsushi/cross:x86_64-unknown-linux-musl"
[target.i686-unknown-linux-gnu]
image = "burntsushi/cross:i686-unknown-linux-gnu"
[target.mips64-unknown-linux-gnuabi64]
image = "burntsushi/cross:mips64-unknown-linux-gnuabi64"
[target.arm-unknown-linux-gnueabihf]
image = "burntsushi/cross:arm-unknown-linux-gnueabihf"

51
FAQ.md
View File

@@ -25,6 +25,7 @@
* [How is ripgrep licensed?](#license)
* [Can ripgrep replace grep?](#posix4ever)
* [What does the "rip" in ripgrep mean?](#intentcountsforsomething)
* [How can I donate to ripgrep or its maintainers?](#donations)
<h3 name="config">
@@ -50,18 +51,19 @@ ripgrep is a project whose contributors are volunteers. A release schedule
adds undue stress to said volunteers. Therefore, releases are made on a best
effort basis and no dates **will ever be given**.
One exception to this is high impact bugs. If a ripgrep release contains a
significant regression, then there will generally be a strong push to get a
patch release out with a fix.
An exception to this _can be_ high impact bugs. If a ripgrep release contains
a significant regression, then there will generally be a strong push to get a
patch release out with a fix. However, no promises are made.
<h3 name="manpage">
Does ripgrep have a man page?
</h3>
Yes! Whenever ripgrep is compiled on a system with `asciidoc` present, then a
man page is generated from ripgrep's argv parser. After compiling ripgrep, you
can find the man page like so from the root of the repository:
Yes! Whenever ripgrep is compiled on a system with `asciidoctor` or `asciidoc`
present, then a man page is generated from ripgrep's argv parser. After
compiling ripgrep, you can find the man page like so from the root of the
repository:
```
$ find ./target -name rg.1 -print0 | xargs -0 ls -t | head -n1
@@ -118,7 +120,7 @@ from run to run of ripgrep.
The only way to make the order of results consistent is to ask ripgrep to
sort the output. Currently, this will disable all parallelism. (On smaller
repositories, you might not notice much of a performance difference!) You
can achieve this with the `--sort-files` flag.
can achieve this with the `--sort path` flag.
There is more discussion on this topic here:
https://github.com/BurntSushi/ripgrep/issues/152
@@ -136,10 +138,10 @@ How do I search compressed files?
</h3>
ripgrep's `-z/--search-zip` flag will cause it to search compressed files
automatically. Currently, this supports gzip, bzip2, lzma, lz4 and xz only and
requires the corresponding `gzip`, `bzip2` and `xz` binaries to be installed on
your system. (That is, ripgrep does decompression by shelling out to another
process.)
automatically. Currently, this supports gzip, bzip2, xz, lzma, lz4, Brotli and
Zstd. Each of these requires requires the corresponding `gzip`, `bzip2`, `xz`,
`lz4`, `brotli` and `zstd` binaries to be installed on your system. (That is,
ripgrep does decompression by shelling out to another process.)
ripgrep currently does not search archive formats, so `*.tar.gz` files, for
example, are skipped.
@@ -149,9 +151,8 @@ example, are skipped.
How do I search over multiple lines?
</h3>
This isn't currently possible. ripgrep is fundamentally a line-oriented search
tool. With that said,
[multiline search is a planned opt-in feature](https://github.com/BurntSushi/ripgrep/issues/176).
The `-U/--multiline` flag enables ripgrep to report results that span over
multiple lines.
<h3 name="fancy">
@@ -935,8 +936,8 @@ Here are some cases where you might *not* want to use ripgrep. The same caveats
for the previous section apply.
* Are you writing portable shell scripts intended to work in a variety of
environments? Great, probably not a good idea to use ripgrep! ripgrep is has
nowhere near the ubquity of grep, so if you do use ripgrep, you might need
environments? Great, probably not a good idea to use ripgrep! ripgrep has
nowhere near the ubiquity of grep, so if you do use ripgrep, you might need
to futz with the installation process more than you would with grep.
* Do you care about POSIX compatibility? If so, then you can't use ripgrep
because it never was, isn't and never will be POSIX compatible.
@@ -982,3 +983,21 @@ grep](#posix4ever),
ripgrep is neither actually a "grep killer" nor was it ever intended to be. It
certainly does eat into some of its use cases, but that's nothing that other
tools like ack or The Silver Searcher weren't already doing.
<h3 name="donations">
How can I donate to ripgrep or its maintainers?
</h3>
As of now, you can't. While I believe the various efforts that are being
undertaken to help fund FOSS are extremely important, they aren't a good fit
for me. ripgrep is and I hope will remain a project of love that I develop in
my free time. As such, involving money---even in the form of donations given
without expectations---would severely change that dynamic for me personally.
Instead, I'd recommend donating to something else that is doing work that you
find meaningful. If you would like suggestions, then my favorites are:
* [The Internet Archive](https://archive.org/donate/)
* [Rails Girls](https://railsgirlssummerofcode.org/campaign/)
* [Wikipedia](https://wikimediafoundation.org/support/)

351
GUIDE.md
View File

@@ -18,6 +18,8 @@ translatable to any command line shell environment.
* [Replacements](#replacements)
* [Configuration file](#configuration-file)
* [File encoding](#file-encoding)
* [Binary data](#binary-data)
* [Preprocessor](#preprocessor)
* [Common options](#common-options)
@@ -109,7 +111,7 @@ colors, you'll notice that `faster` will be highlighted instead of just the
It is beyond the scope of this guide to provide a full tutorial on regular
expressions, but ripgrep's specific syntax is documented here:
https://docs.rs/regex/0.2.5/regex/#syntax
https://docs.rs/regex/*/regex/#syntax
### Recursive search
@@ -235,6 +237,11 @@ Like `.gitignore`, a `.ignore` file can be placed in any directory. Its rules
will be processed with respect to the directory it resides in, just like
`.gitignore`.
To process `.gitignore` and `.ignore` files case insensitively, use the flag
`--ignore-file-case-insensitive`. This is especially useful on case insensitive
file systems like those on Windows and macOS. Note though that this can come
with a significant performance penalty, and is therefore disabled by default.
For a more in depth description of how glob patterns in a `.gitignore` file
are interpreted, please see `man gitignore`.
@@ -405,6 +412,21 @@ alias rg="rg --type-add 'web:*.{html,css,js}'"
or add `--type-add=web:*.{html,css,js}` to your ripgrep configuration file.
([Configuration files](#configuration-file) are covered in more detail later.)
#### The special `all` file type
A special option supported by the `--type` flag is `all`. `--type all` looks
for a match in any of the supported file types listed by `--type-list`,
including those added on the command line using `--type-add`. It's equivalent
to the command `rg --type agda --type asciidoc --type asm ...`, where `...`
stands for a list of `--type` flags for the rest of the types in `--type-list`.
As an example, let's suppose you have a shell script in your current directory,
`my-shell-script`, which includes a shell library, `my-shell-library.bash`.
Both `rg --type sh` and `rg --type all` would only search for matches in
`my-shell-library.bash`, not `my-shell-script`, because the globs matched
by the `sh` file type don't include files without an extension. On the
other hand, `rg --type-not all` would search `my-shell-script` but not
`my-shell-library.bash`.
### Replacements
@@ -520,9 +542,9 @@ config file. Once the environment variable is set, open the file and just type
in the flags you want set automatically. There are only two rules for
describing the format of the config file:
1. Every line is a shell argument, after trimming ASCII whitespace.
2. Lines starting with `#` (optionally preceded by any amount of
ASCII whitespace) are ignored.
1. Every line is a shell argument, after trimming whitespace.
2. Lines starting with `#` (optionally preceded by any amount of whitespace)
are ignored.
In particular, there is no escaping. Each line is given to ripgrep as a single
command line argument verbatim.
@@ -532,8 +554,9 @@ formatting peculiarities:
```
$ cat $HOME/.ripgreprc
# Don't let ripgrep vomit really long lines to my terminal.
# Don't let ripgrep vomit really long lines to my terminal, and show a preview.
--max-columns=150
--max-columns-preview
# Add my 'web' type.
--type-add
@@ -598,13 +621,14 @@ topic, but we can try to summarize its relevancy to ripgrep:
* Files are generally just a bundle of bytes. There is no reliable way to know
their encoding.
* Either the encoding of the pattern must match the encoding of the files being
searched, or a form of transcoding must be performed converts either the
searched, or a form of transcoding must be performed that converts either the
pattern or the file to the same encoding as the other.
* ripgrep tends to work best on plain text files, and among plain text files,
the most popular encodings likely consist of ASCII, latin1 or UTF-8. As
a special exception, UTF-16 is prevalent in Windows environments
In light of the above, here is how ripgrep behaves:
In light of the above, here is how ripgrep behaves when `--encoding auto` is
given, which is the default:
* All input is assumed to be ASCII compatible (which means every byte that
corresponds to an ASCII codepoint actually is an ASCII codepoint). This
@@ -620,12 +644,15 @@ In light of the above, here is how ripgrep behaves:
they correspond to a UTF-16 BOM, then ripgrep will transcode the contents of
the file from UTF-16 to UTF-8, and then execute the search on the transcoded
version of the file. (This incurs a performance penalty since transcoding
is slower than regex searching.)
is slower than regex searching.) If the file contains invalid UTF-16, then
the Unicode replacement codepoint is substituted in place of invalid code
units.
* To handle other cases, ripgrep provides a `-E/--encoding` flag, which permits
you to specify an encoding from the
[Encoding Standard](https://encoding.spec.whatwg.org/#concept-encoding-get).
ripgrep will assume *all* files searched are the encoding specified and
will perform a transcoding step just like in the UTF-16 case described above.
ripgrep will assume *all* files searched are the encoding specified (unless
the file has a BOM) and will perform a transcoding step just like in the
UTF-16 case described above.
By default, ripgrep will not require its input be valid UTF-8. That is, ripgrep
can and will search arbitrary bytes. The key here is that if you're searching
@@ -635,9 +662,26 @@ pattern won't find anything. With all that said, this mode of operation is
important, because it lets you find ASCII or UTF-8 *within* files that are
otherwise arbitrary bytes.
As a special case, the `-E/--encoding` flag supports the value `none`, which
will completely disable all encoding related logic, including BOM sniffing.
When `-E/--encoding` is set to `none`, ripgrep will search the raw bytes of
the underlying file with no transcoding step. For example, here's how you might
search the raw UTF-16 encoding of the string `Шерлок`:
```
$ rg '(?-u)\(\x045\x04@\x04;\x04>\x04:\x04' -E none -a some-utf16-file
```
Of course, that's just an example meant to show how one can drop down into
raw bytes. Namely, the simpler command works as you might expect automatically:
```
$ rg 'Шерлок' some-utf16-file
```
Finally, it is possible to disable ripgrep's Unicode support from within the
pattern regular expression. For example, let's say you wanted `.` to match any
byte rather than any Unicode codepoint. (You might want this while searching a
regular expression. For example, let's say you wanted `.` to match any byte
rather than any Unicode codepoint. (You might want this while searching a
binary file, since `.` by default will not match invalid UTF-8.) You could do
this by disabling Unicode via a regular expression flag:
@@ -654,6 +698,282 @@ $ rg '\w(?-u:\w)\w'
```
### Binary data
In addition to skipping hidden files and files in your `.gitignore` by default,
ripgrep also attempts to skip binary files. ripgrep does this by default
because binary files (like PDFs or images) are typically not things you want to
search when searching for regex matches. Moreover, if content in a binary file
did match, then it's possible for undesirable binary data to be printed to your
terminal and wreak havoc.
Unfortunately, unlike skipping hidden files and respecting your `.gitignore`
rules, a file cannot as easily be classified as binary. In order to figure out
whether a file is binary, the most effective heuristic that balances
correctness with performance is to simply look for `NUL` bytes. At that point,
the determination is simple: a file is considered "binary" if and only if it
contains a `NUL` byte somewhere in its contents.
The issue is that while most binary files will have a `NUL` byte toward the
beginning of its contents, this is not necessarily true. The `NUL` byte might
be the very last byte in a large file, but that file is still considered
binary. While this leads to a fair amount of complexity inside ripgrep's
implementation, it also results in some unintuitive user experiences.
At a high level, ripgrep operates in three different modes with respect to
binary files:
1. The default mode is to attempt to remove binary files from a search
completely. This is meant to mirror how ripgrep removes hidden files and
files in your `.gitignore` automatically. That is, as soon as a file is
detected as binary, searching stops. If a match was already printed (because
it was detected long before a `NUL` byte), then ripgrep will print a warning
message indicating that the search stopped prematurely. This default mode
**only applies to files searched by ripgrep as a result of recursive
directory traversal**, which is consistent with ripgrep's other automatic
filtering. For example, `rg foo .file` will search `.file` even though it
is hidden. Similarly, `rg foo binary-file` will search `binary-file` in
"binary" mode automatically.
2. Binary mode is similar to the default mode, except it will not always
stop searching after it sees a `NUL` byte. Namely, in this mode, ripgrep
will continue searching a file that is known to be binary until the first
of two conditions is met: 1) the end of the file has been reached or 2) a
match is or has been seen. This means that in binary mode, if ripgrep
reports no matches, then there are no matches in the file. When a match does
occur, ripgrep prints a message similar to one it prints when in its default
mode indicating that the search has stopped prematurely. This mode can be
forcefully enabled for all files with the `--binary` flag. The purpose of
binary mode is to provide a way to discover matches in all files, but to
avoid having binary data dumped into your terminal.
3. Text mode completely disables all binary detection and searches all files
as if they were text. This is useful when searching a file that is
predominantly text but contains a `NUL` byte, or if you are specifically
trying to search binary data. This mode can be enabled with the `-a/--text`
flag. Note that when using this mode on very large binary files, it is
possible for ripgrep to use a lot of memory.
Unfortunately, there is one additional complexity in ripgrep that can make it
difficult to reason about binary files. That is, the way binary detection works
depends on the way that ripgrep searches your files. Specifically:
* When ripgrep uses memory maps, then binary detection is only performed on the
first few kilobytes of the file in addition to every matching line.
* When ripgrep doesn't use memory maps, then binary detection is performed on
all bytes searched.
This means that whether a file is detected as binary or not can change based
on the internal search strategy used by ripgrep. If you prefer to keep
ripgrep's binary file detection consistent, then you can disable memory maps
via the `--no-mmap` flag. (The cost will be a small performance regression when
searching very large files on some platforms.)
### Preprocessor
In ripgrep, a preprocessor is any type of command that can be run to transform
the input of every file before ripgrep searches it. This makes it possible to
search virtually any kind of content that can be automatically converted to
text without having to teach ripgrep how to read said content.
One common example is searching PDFs. PDFs are first and foremost meant to be
displayed to users. But PDFs often have text streams in them that can be useful
to search. In our case, we want to search Bruce Watson's excellent
dissertation,
[Taxonomies and Toolkits of Regular Language Algorithms](https://burntsushi.net/stuff/1995-watson.pdf).
After downloading it, let's try searching it:
```
$ rg 'The Commentz-Walter algorithm' 1995-watson.pdf
$
```
Surely, a dissertation on regular language algorithms would mention
Commentz-Walter. Indeed it does, but our search isn't picking it up because
PDFs are a binary format, and the text shown in the PDF may not be encoded as
simple contiguous UTF-8. Namely, even passing the `-a/--text` flag to ripgrep
will not make our search work.
One way to fix this is to convert the PDF to plain text first. This won't work
well for all PDFs, but does great in a lot of cases. (Note that the tool we
use, `pdftotext`, is part of the [poppler](https://poppler.freedesktop.org)
PDF rendering library.)
```
$ pdftotext 1995-watson.pdf > 1995-watson.txt
$ rg 'The Commentz-Walter algorithm' 1995-watson.txt
316:The Commentz-Walter algorithms : : : : : : : : : : : : : : :
7165:4.4 The Commentz-Walter algorithms
10062:in input string S , we obtain the Boyer-Moore algorithm. The Commentz-Walter algorithm
17218:The Commentz-Walter algorithm (and its variants) displayed more interesting behaviour,
17249:Aho-Corasick algorithms are used extensively. The Commentz-Walter algorithms are used
17297: The Commentz-Walter algorithms (CW). In all versions of the CW algorithms, a common program skeleton is used with di erent shift functions. The CW algorithms are
```
But having to explicitly convert every file can be a pain, especially when you
have a directory full of PDF files. Instead, we can use ripgrep's preprocessor
feature to search the PDF. ripgrep's `--pre` flag works by taking a single
command name and then executing that command for every file that it searches.
ripgrep passes the file path as the first and only argument to the command and
also sends the contents of the file to stdin. So let's write a simple shell
script that wraps `pdftotext` in a way that conforms to this interface:
```
$ cat preprocess
#!/bin/sh
exec pdftotext - -
```
With `preprocess` in the same directory as `1995-watson.pdf`, we can now use it
to search the PDF:
```
$ rg --pre ./preprocess 'The Commentz-Walter algorithm' 1995-watson.pdf
316:The Commentz-Walter algorithms : : : : : : : : : : : : : : :
7165:4.4 The Commentz-Walter algorithms
10062:in input string S , we obtain the Boyer-Moore algorithm. The Commentz-Walter algorithm
17218:The Commentz-Walter algorithm (and its variants) displayed more interesting behaviour,
17249:Aho-Corasick algorithms are used extensively. The Commentz-Walter algorithms are used
17297: The Commentz-Walter algorithms (CW). In all versions of the CW algorithms, a common program skeleton is used with di erent shift functions. The CW algorithms are
```
Note that `preprocess` must be resolvable to a command that ripgrep can read.
The simplest way to do this is to put your preprocessor command in a directory
that is in your `PATH` (or equivalent), or otherwise use an absolute path.
As a bonus, this turns out to be quite a bit faster than other specialized PDF
grepping tools:
```
$ time rg --pre ./preprocess 'The Commentz-Walter algorithm' 1995-watson.pdf -c
6
real 0.697
user 0.684
sys 0.007
maxmem 16 MB
faults 0
$ time pdfgrep 'The Commentz-Walter algorithm' 1995-watson.pdf -c
6
real 1.336
user 1.310
sys 0.023
maxmem 16 MB
faults 0
```
If you wind up needing to search a lot of PDFs, then ripgrep's parallelism can
make the speed difference even greater.
#### A more robust preprocessor
One of the problems with the aforementioned preprocessor is that it will fail
if you try to search a file that isn't a PDF:
```
$ echo foo > not-a-pdf
$ rg --pre ./preprocess 'The Commentz-Walter algorithm' not-a-pdf
not-a-pdf: preprocessor command failed: '"./preprocess" "not-a-pdf"':
-------------------------------------------------------------------------------
Syntax Warning: May not be a PDF file (continuing anyway)
Syntax Error: Couldn't find trailer dictionary
Syntax Error: Couldn't find trailer dictionary
Syntax Error: Couldn't read xref table
```
To fix this, we can make our preprocessor script a bit more robust by only
running `pdftotext` when we think the input is a non-empty PDF:
```
$ cat preprocessor
#!/bin/sh
case "$1" in
*.pdf)
# The -s flag ensures that the file is non-empty.
if [ -s "$1" ]; then
exec pdftotext - -
else
exec cat
fi
;;
*)
exec cat
;;
esac
```
We can even extend our preprocessor to search other kinds of files. Sometimes
we don't always know the file type from the file name, so we can use the `file`
utility to "sniff" the type of the file based on its contents:
```
$ cat processor
#!/bin/sh
case "$1" in
*.pdf)
# The -s flag ensures that the file is non-empty.
if [ -s "$1" ]; then
exec pdftotext - -
else
exec cat
fi
;;
*)
case $(file "$1") in
*Zstandard*)
exec pzstd -cdq
;;
*)
exec cat
;;
esac
;;
esac
```
#### Reducing preprocessor overhead
There is one more problem with the above approach: it requires running a
preprocessor for every single file that ripgrep searches. If every file needs
a preprocessor, then this is OK. But if most don't, then this can substantially
slow down searches because of the overhead of launching new processors. You
can avoid this by telling ripgrep to only invoke the preprocessor when the file
path matches a glob. For example, consider the performance difference even when
searching a repository as small as ripgrep's:
```
$ time rg --pre pre-rg 'fn is_empty' -c
crates/globset/src/lib.rs:1
crates/matcher/src/lib.rs:2
crates/ignore/src/overrides.rs:1
crates/ignore/src/gitignore.rs:1
crates/ignore/src/types.rs:1
real 0.138
user 0.485
sys 0.209
maxmem 7 MB
faults 0
$ time rg --pre pre-rg --pre-glob '*.pdf' 'fn is_empty' -c
crates/globset/src/lib.rs:1
crates/ignore/src/types.rs:1
crates/ignore/src/gitignore.rs:1
crates/ignore/src/overrides.rs:1
crates/matcher/src/lib.rs:2
real 0.008
user 0.010
sys 0.002
maxmem 7 MB
faults 0
```
### Common options
ripgrep has a lot of flags. Too many to keep in your head at once. This section
@@ -675,10 +995,11 @@ used options that will likely impact how you use ripgrep on a regular basis.
* `--files`: Print the files that ripgrep *would* search, but don't actually
search them.
* `-a/--text`: Search binary files as if they were plain text.
* `-z/--search-zip`: Search compressed files (gzip, bzip2, lzma, xz). This is
disabled by default.
* `-U/--multiline`: Permit matches to span multiple lines.
* `-z/--search-zip`: Search compressed files (gzip, bzip2, lzma, xz, lz4,
brotli, zstd). This is disabled by default.
* `-C/--context`: Show the lines surrounding a match.
* `--sort-files`: Force ripgrep to sort its output by file name. (This disables
* `--sort path`: Force ripgrep to sort its output by file name. (This disables
parallelism, so it might be slower.)
* `-L/--follow`: Follow symbolic links while recursively searching.
* `-M/--max-columns`: Limit the length of lines printed by ripgrep.

204
README.md
View File

@@ -1,17 +1,18 @@
ripgrep (rg)
------------
ripgrep is a line-oriented search tool that recursively searches your current
directory for a regex pattern while respecting your gitignore rules. ripgrep
directory for a regex pattern. By default, ripgrep will respect your .gitignore
and automatically skip hidden files/directories and binary files. ripgrep
has first class support on Windows, macOS and Linux, with binary downloads
available for [every release](https://github.com/BurntSushi/ripgrep/releases).
ripgrep is similar to other popular search tools like The Silver Searcher,
ack and grep.
ripgrep is similar to other popular search tools like The Silver Searcher, ack
and grep.
[![Linux build status](https://travis-ci.org/BurntSushi/ripgrep.svg)](https://travis-ci.org/BurntSushi/ripgrep)
[![Windows build status](https://ci.appveyor.com/api/projects/status/github/BurntSushi/ripgrep?svg=true)](https://ci.appveyor.com/project/BurntSushi/ripgrep)
[![Build status](https://github.com/BurntSushi/ripgrep/workflows/ci/badge.svg)](https://github.com/BurntSushi/ripgrep/actions)
[![Crates.io](https://img.shields.io/crates/v/ripgrep.svg)](https://crates.io/crates/ripgrep)
[![Packaging status](https://repology.org/badge/tiny-repos/ripgrep.svg)](https://repology.org/project/ripgrep/badges)
Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).
Dual-licensed under MIT or the [UNLICENSE](https://unlicense.org).
### CHANGELOG
@@ -27,60 +28,60 @@ Please see the [CHANGELOG](CHANGELOG.md) for a release history.
* [Configuration files](GUIDE.md#configuration-file)
* [Shell completions](FAQ.md#complete)
* [Building](#building)
* [Translations](#translations)
### Screenshot of search results
[![A screenshot of a sample search with ripgrep](http://burntsushi.net/stuff/ripgrep1.png)](http://burntsushi.net/stuff/ripgrep1.png)
[![A screenshot of a sample search with ripgrep](https://burntsushi.net/stuff/ripgrep1.png)](https://burntsushi.net/stuff/ripgrep1.png)
### Quick examples comparing tools
This example searches the entire Linux kernel source tree (after running
`make defconfig && make -j8`) for `[A-Z]+_SUSPEND`, where all matches must be
words. Timings were collected on a system with an Intel i7-6900K 3.2 GHz, and
ripgrep was compiled with SIMD enabled.
This example searches the entire
[Linux kernel source tree](https://github.com/BurntSushi/linux)
(after running `make defconfig && make -j8`) for `[A-Z]+_SUSPEND`, where
all matches must be words. Timings were collected on a system with an Intel
i7-6900K 3.2 GHz.
Please remember that a single benchmark is never enough! See my
[blog post on ripgrep](http://blog.burntsushi.net/ripgrep/)
[blog post on ripgrep](https://blog.burntsushi.net/ripgrep/)
for a very detailed comparison with more benchmarks and analysis.
| Tool | Command | Line count | Time |
| ---- | ------- | ---------- | ---- |
| ripgrep (Unicode) | `rg -n -w '[A-Z]+_SUSPEND'` | 450 | **0.106s** |
| [git grep](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=C git grep -E -n -w '[A-Z]+_SUSPEND'` | 450 | 0.553s |
| [The Silver Searcher](https://github.com/ggreer/the_silver_searcher) | `ag -w '[A-Z]+_SUSPEND'` | 450 | 0.589s |
| [git grep (Unicode)](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=en_US.UTF-8 git grep -E -n -w '[A-Z]+_SUSPEND'` | 450 | 2.266s |
| [sift](https://github.com/svent/sift) | `sift --git -n -w '[A-Z]+_SUSPEND'` | 450 | 3.505s |
| [ack](https://github.com/petdance/ack2) | `ack -w '[A-Z]+_SUSPEND'` | 1878 | 6.823s |
| [The Platinum Searcher](https://github.com/monochromegane/the_platinum_searcher) | `pt -w -e '[A-Z]+_SUSPEND'` | 450 | 14.208s |
| ripgrep (Unicode) | `rg -n -w '[A-Z]+_SUSPEND'` | 452 | **0.136s** |
| [git grep](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `git grep -P -n -w '[A-Z]+_SUSPEND'` | 452 | 0.348s |
| [ugrep (Unicode)](https://github.com/Genivia/ugrep) | `ugrep -r --ignore-files --no-hidden -I -w '[A-Z]+_SUSPEND'` | 452 | 0.506s |
| [The Silver Searcher](https://github.com/ggreer/the_silver_searcher) | `ag -w '[A-Z]+_SUSPEND'` | 452 | 0.654s |
| [git grep](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=C git grep -E -n -w '[A-Z]+_SUSPEND'` | 452 | 1.150s |
| [ack](https://github.com/beyondgrep/ack3) | `ack -w '[A-Z]+_SUSPEND'` | 452 | 4.054s |
| [git grep (Unicode)](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=en_US.UTF-8 git grep -E -n -w '[A-Z]+_SUSPEND'` | 452 | 4.205s |
(Yes, `ack` [has](https://github.com/petdance/ack2/issues/445) a
[bug](https://github.com/petdance/ack2/issues/14).)
Here's another benchmark that disregards gitignore files and searches with a
whitelist instead. The corpus is the same as in the previous benchmark, and the
flags passed to each command ensure that they are doing equivalent work:
Here's another benchmark on the same corpus as above that disregards gitignore
files and searches with a whitelist instead. The corpus is the same as in the
previous benchmark, and the flags passed to each command ensure that they are
doing equivalent work:
| Tool | Command | Line count | Time |
| ---- | ------- | ---------- | ---- |
| ripgrep | `rg -L -u -tc -n -w '[A-Z]+_SUSPEND'` | 404 | **0.079s** |
| [ucg](https://github.com/gvansickle/ucg) | `ucg --type=cc -w '[A-Z]+_SUSPEND'` | 390 | 0.163s |
| [GNU grep](https://www.gnu.org/software/grep/) | `egrep -R -n --include='*.c' --include='*.h' -w '[A-Z]+_SUSPEND'` | 404 | 0.611s |
| ripgrep | `rg -uuu -tc -n -w '[A-Z]+_SUSPEND'` | 388 | **0.096s** |
| [ugrep](https://github.com/Genivia/ugrep) | `ugrep -r -n --include='*.c' --include='*.h' -w '[A-Z]+_SUSPEND'` | 388 | 0.493s |
| [GNU grep](https://www.gnu.org/software/grep/) | `egrep -r -n --include='*.c' --include='*.h' -w '[A-Z]+_SUSPEND'` | 388 | 0.806s |
(`ucg` [has slightly different behavior in the presence of symbolic links](https://github.com/gvansickle/ucg/issues/106).)
And finally, a straight-up comparison between ripgrep and GNU grep on a single
large file (~9.3GB,
[`OpenSubtitles2016.raw.en.gz`](http://opus.lingfil.uu.se/OpenSubtitles2016/mono/OpenSubtitles2016.raw.en.gz)):
And finally, a straight-up comparison between ripgrep, ugrep and GNU grep on a
single large file cached in memory
(~13GB, [`OpenSubtitles.raw.en.gz`](http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2018/mono/OpenSubtitles.raw.en.gz)):
| Tool | Command | Line count | Time |
| ---- | ------- | ---------- | ---- |
| ripgrep | `rg -w 'Sherlock [A-Z]\w+'` | 5268 | **2.108s** |
| [GNU grep](https://www.gnu.org/software/grep/) | `LC_ALL=C egrep -w 'Sherlock [A-Z]\w+'` | 5268 | 7.014s |
| ripgrep | `rg -w 'Sherlock [A-Z]\w+'` | 7882 | **2.769s** |
| [ugrep](https://github.com/Genivia/ugrep) | `ugrep -w 'Sherlock [A-Z]\w+'` | 7882 | 6.802s |
| [GNU grep](https://www.gnu.org/software/grep/) | `LC_ALL=en_US.UTF-8 egrep -w 'Sherlock [A-Z]\w+'` | 7882 | 9.027s |
In the above benchmark, passing the `-n` flag (for showing line numbers)
increases the times to `2.640s` for ripgrep and `10.277s` for GNU grep.
increases the times to `3.423s` for ripgrep and `13.031s` for GNU grep. ugrep
times are unaffected by the presence or absence of `-n`.
### Why should I use ripgrep?
@@ -90,11 +91,11 @@ increases the times to `2.640s` for ripgrep and `10.277s` for GNU grep.
[the FAQ](FAQ.md#posix4ever) for more details on whether ripgrep can truly
replace grep.)
* Like other tools specialized to code search, ripgrep defaults to recursive
directory search and won't search files ignored by your `.gitignore` files.
It also ignores hidden and binary files by default. ripgrep also implements
full support for `.gitignore`, whereas there are many bugs related to that
functionality in other code search tools claiming to provide the same
functionality.
directory search and won't search files ignored by your
`.gitignore`/`.ignore`/`.rgignore` files. It also ignores hidden and binary
files by default. ripgrep also implements full support for `.gitignore`,
whereas there are many bugs related to that functionality in other code
search tools claiming to provide the same functionality.
* ripgrep can search specific types of files. For example, `rg -tpy foo`
limits your search to Python files and `rg -Tjs foo` excludes Javascript
files from your search. ripgrep can be taught about new file types with
@@ -105,17 +106,20 @@ increases the times to `2.640s` for ripgrep and `10.277s` for GNU grep.
supporting Unicode (which is always on).
* ripgrep has optional support for switching its regex engine to use PCRE2.
Among other things, this makes it possible to use look-around and
backreferences in your patterns, which are supported in ripgrep's default
regex engine. PCRE2 support is enabled with `-P`.
backreferences in your patterns, which are not supported in ripgrep's default
regex engine. PCRE2 support can be enabled with `-P/--pcre2` (use PCRE2
always) or `--auto-hybrid-regex` (use PCRE2 only if needed). An alternative
syntax is provided via the `--engine (default|pcre2|auto-hybrid)` option.
* ripgrep supports searching files in text encodings other than UTF-8, such
as UTF-16, latin-1, GBK, EUC-JP, Shift_JIS and more. (Some support for
automatically detecting UTF-16 is provided. Other text encodings must be
specifically specified with the `-E/--encoding` flag.)
* ripgrep supports searching files compressed in a common format (gzip, xz,
lzma, bzip2 or lz4) with the `-z/--search-zip` flag.
* ripgrep supports arbitrary input preprocessing filters which could be PDF
text extraction, less supported decompression, decrypting, automatic encoding
detection and so on.
* ripgrep supports searching files compressed in a common format (brotli,
bzip2, gzip, lz4, lzma, xz, or zstandard) with the `-z/--search-zip` flag.
* ripgrep supports
[arbitrary input preprocessing filters](GUIDE.md#preprocessor)
which could be PDF text extraction, less supported decompression, decrypting,
automatic encoding detection and so on.
In other words, use ripgrep if you like speed, filtering by default, fewer
bugs and Unicode support.
@@ -146,12 +150,12 @@ or more of the following:
### Is it really faster than everything else?
Generally, yes. A large number of benchmarks with detailed analysis for each is
[available on my blog](http://blog.burntsushi.net/ripgrep/).
[available on my blog](https://blog.burntsushi.net/ripgrep/).
Summarizing, ripgrep is fast because:
* It is built on top of
[Rust's regex engine](https://github.com/rust-lang-nursery/regex).
[Rust's regex engine](https://github.com/rust-lang/regex).
Rust's regex engine uses finite automata, SIMD and aggressive literal
optimizations to make searching very fast. (PCRE2 support can be opted into
with the `-P/--pcre2` flag.)
@@ -198,22 +202,13 @@ prefer MSVC over GNU, but you'll need to have the [Microsoft VC++ 2015
redistributable](https://www.microsoft.com/en-us/download/details.aspx?id=48145)
installed.
If you're a **macOS Homebrew** or a **Linuxbrew** user,
then you can install ripgrep either
from homebrew-core, (compiled with rust stable, no SIMD):
If you're a **macOS Homebrew** or a **Linuxbrew** user, then you can install
ripgrep from homebrew-core:
```
$ brew install ripgrep
```
or you can install a binary compiled with rust nightly (including SIMD and all
optimizations) by utilizing a custom tap:
```
$ brew tap burntsushi/ripgrep https://github.com/BurntSushi/ripgrep.git
$ brew install ripgrep-bin
```
If you're a **MacPorts** user, then you can install ripgrep from the
[official ports](https://www.macports.org/ports.php?by=name&substr=ripgrep):
@@ -229,7 +224,7 @@ $ choco install ripgrep
```
If you're a **Windows Scoop** user, then you can install ripgrep from the
[official bucket](https://github.com/lukesampson/scoop/blob/master/bucket/ripgrep.json):
[official bucket](https://github.com/ScoopInstaller/Main/blob/master/bucket/ripgrep.json):
```
$ scoop install ripgrep
@@ -248,29 +243,21 @@ If you're a **Gentoo** user, you can install ripgrep from the
$ emerge sys-apps/ripgrep
```
If you're a **Fedora 27+** user, you can install ripgrep from official
If you're a **Fedora** user, you can install ripgrep from official
repositories.
```
$ sudo dnf install ripgrep
```
If you're a **Fedora 24+** user, you can install ripgrep from
[copr](https://copr.fedorainfracloud.org/coprs/carlwgeorge/ripgrep/):
```
$ sudo dnf copr enable carlwgeorge/ripgrep
$ sudo dnf install ripgrep
```
If you're an **openSUSE Tumbleweed** user, you can install ripgrep from the
[official repo](http://software.opensuse.org/package/ripgrep):
If you're an **openSUSE** user, ripgrep is included in **openSUSE Tumbleweed**
and **openSUSE Leap** since 15.1.
```
$ sudo zypper install ripgrep
```
If you're a **RHEL/CentOS 7** user, you can install ripgrep from
If you're a **RHEL/CentOS 7/8** user, you can install ripgrep from
[copr](https://copr.fedorainfracloud.org/coprs/carlwgeorge/ripgrep/):
```
@@ -288,20 +275,27 @@ $ # (Or using the attribute name, which is also ripgrep.)
If you're a **Debian** user (or a user of a Debian derivative like **Ubuntu**),
then ripgrep can be installed using a binary `.deb` file provided in each
[ripgrep release](https://github.com/BurntSushi/ripgrep/releases). Note that
ripgrep is not in the official Debian or Ubuntu repositories.
[ripgrep release](https://github.com/BurntSushi/ripgrep/releases).
```
$ curl -LO https://github.com/BurntSushi/ripgrep/releases/download/0.9.0/ripgrep_0.9.0_amd64.deb
$ sudo dpkg -i ripgrep_0.9.0_amd64.deb
$ curl -LO https://github.com/BurntSushi/ripgrep/releases/download/11.0.2/ripgrep_11.0.2_amd64.deb
$ sudo dpkg -i ripgrep_11.0.2_amd64.deb
```
If you run Debian Buster (currently Debian testing) or Debian sid, ripgrep is
If you run Debian Buster (currently Debian stable) or Debian sid, ripgrep is
[officially maintained by Debian](https://tracker.debian.org/pkg/rust-ripgrep).
```
$ sudo apt-get install ripgrep
```
If you're an **Ubuntu Cosmic (18.10)** (or newer) user, ripgrep is
[available](https://launchpad.net/ubuntu/+source/rust-ripgrep) using the same
packaging as Debian:
```
$ sudo apt-get install ripgrep
```
(N.B. Various snaps for ripgrep on Ubuntu are also available, but none of them
seem to work right and generate a number of very strange bug reports that I
don't know how to fix and don't have the time to fix. Therefore, it is no
@@ -328,9 +322,23 @@ If you're a **NetBSD** user, then you can install ripgrep from
# pkgin install ripgrep
```
If you're a **Haiku x86_64** user, then you can install ripgrep from the
[official ports](https://github.com/haikuports/haikuports/tree/master/sys-apps/ripgrep):
```
$ pkgman install ripgrep
```
If you're a **Haiku x86_gcc2** user, then you can install ripgrep from the
same port as Haiku x86_64 using the x86 secondary architecture build:
```
$ pkgman install ripgrep_x86
```
If you're a **Rust programmer**, ripgrep can be installed with `cargo`.
* Note that the minimum supported version of Rust for ripgrep is **1.28.0**,
* Note that the minimum supported version of Rust for ripgrep is **1.34.0**,
although ripgrep may work with older versions.
* Note that the binary may be bigger than expected because it contains debug
symbols. This is intentional. To remove debug symbols and therefore reduce
@@ -340,18 +348,12 @@ If you're a **Rust programmer**, ripgrep can be installed with `cargo`.
$ cargo install ripgrep
```
When compiling with Rust 1.27 or newer, this will automatically enable SIMD
optimizations for search.
ripgrep isn't currently in any other package repositories.
[I'd like to change that](https://github.com/BurntSushi/ripgrep/issues/10).
### Building
ripgrep is written in Rust, so you'll need to grab a
[Rust installation](https://www.rust-lang.org/) in order to compile it.
ripgrep compiles with Rust 1.28.0 (stable) or newer. In general, ripgrep tracks
ripgrep compiles with Rust 1.34.0 (stable) or newer. In general, ripgrep tracks
the latest stable release of the Rust compiler.
To build ripgrep:
@@ -368,18 +370,14 @@ If you have a Rust nightly compiler and a recent Intel CPU, then you can enable
additional optional SIMD acceleration like so:
```
RUSTFLAGS="-C target-cpu=native" cargo build --release --features 'simd-accel avx-accel'
RUSTFLAGS="-C target-cpu=native" cargo build --release --features 'simd-accel'
```
If your machine doesn't support AVX instructions, then simply remove
`avx-accel` from the features list. Similarly for SIMD (which corresponds
roughly to SSE instructions).
The `simd-accel` and `avx-accel` features enable SIMD support in certain
ripgrep dependencies (responsible for counting lines and transcoding). They
are not necessary to get SIMD optimizations for search; those are enabled
automatically. Hopefully, some day, the `simd-accel` and `avx-accel` features
will similarly become unnecessary.
The `simd-accel` feature enables SIMD support in certain ripgrep dependencies
(responsible for transcoding). They are not necessary to get SIMD optimizations
for search; those are enabled automatically. Hopefully, some day, the
`simd-accel` feature will similarly become unnecessary. **WARNING:** Currently,
enabling this option can increase compilation times dramatically.
Finally, optional PCRE2 support can be built with ripgrep by enabling the
`pcre2` feature:
@@ -388,8 +386,8 @@ Finally, optional PCRE2 support can be built with ripgrep by enabling the
$ cargo build --release --features 'pcre2'
```
(Tip: use `--features 'pcre2 simd-accel avx-accel'` to also include compile
time SIMD optimizations, which will only work with a nightly compiler.)
(Tip: use `--features 'pcre2 simd-accel'` to also include compile time SIMD
optimizations, which will only work with a nightly compiler.)
Enabling the PCRE2 feature works with a stable Rust compiler and will
attempt to automatically find and link with your system's PCRE2 library via
@@ -425,3 +423,11 @@ $ cargo test --all
```
from the repository root.
### Translations
The following is a list of known translations of ripgrep's documentation. These
are unofficially maintained and may not be up to date.
* [Chinese](https://github.com/chinanf-boy/ripgrep-zh#%E6%9B%B4%E6%96%B0-)

43
RELEASE-CHECKLIST.md Normal file
View File

@@ -0,0 +1,43 @@
Release Checklist
-----------------
* Run `cargo update` and review dependency updates. Commit updated
`Cargo.lock`.
* Run `cargo outdated` and review semver incompatible updates. Unless there is
a strong motivation otherwise, review and update every dependency.
* Review changes for every crate in `crates` since the last ripgrep release.
If the set of changes is non-empty, issue a new release for that crate. Check
crates in the following order. After updating a crate, ensure minimal
versions are updated as appropriate in dependents. If an update is required,
run `cargo-up --no-push crates/{CRATE}/Cargo.toml`.
* crates/globset
* crates/ignore
* crates/cli
* crates/matcher
* crates/regex
* crates/pcre2
* crates/searcher
* crates/printer
* crates/grep (bump minimal versions as necessary)
* crates/core (do **not** bump version, but update dependencies as needed)
* Update the CHANGELOG as appropriate.
* Edit the `Cargo.toml` to set the new ripgrep version. Run
`cargo update -p ripgrep` so that the `Cargo.lock` is updated. Commit the
changes and create a new signed tag. Alternatively, use
`cargo-up --no-push --no-release Cargo.toml {VERSION}` to automate this.
* Wait for CI to finish creating the release. If the release build fails, then
delete the tag from GitHub, make fixes, re-tag, delete the release and push.
* Copy the relevant section of the CHANGELOG to the tagged release notes.
Include this blurb describing what ripgrep is:
> In case you haven't heard of it before, ripgrep is a line-oriented search
> tool that recursively searches your current directory for a regex pattern.
> By default, ripgrep will respect your gitignore rules and automatically
> skip hidden files/directories and binary files.
* Run `ci/build-deb` locally and manually upload the deb package to the
release.
* Run `cargo publish`.
* Run `ci/sha256-releases {VERSION} >> pkg/brew/ripgrep-bin.rb`. Then edit
`pkg/brew/ripgrep-bin.rb` to update the version number and sha256 hashes.
Remove extraneous stuff added by `ci/sha256-releases`. Commit changes.
Note that
[`cargo-up` can be found in BurntSushi's dotfiles](https://github.com/BurntSushi/dotfiles/blob/master/bin/cargo-up).

81
appveyor.yml
View File

@@ -1,81 +0,0 @@
cache:
- c:\cargo\registry
- c:\cargo\git
init:
- mkdir c:\cargo
- mkdir c:\rustup
- SET PATH=c:\cargo\bin;%PATH%
clone_folder: c:\projects\ripgrep
environment:
CARGO_HOME: "c:\\cargo"
RUSTUP_HOME: "c:\\rustup"
CARGO_TARGET_DIR: "c:\\projects\\ripgrep\\target"
global:
PROJECT_NAME: ripgrep
RUST_BACKTRACE: full
matrix:
- TARGET: x86_64-pc-windows-gnu
CHANNEL: stable
BITS: 64
MSYS2: 1
- TARGET: x86_64-pc-windows-msvc
CHANNEL: stable
BITS: 64
- TARGET: i686-pc-windows-gnu
CHANNEL: stable
BITS: 32
MSYS2: 1
- TARGET: i686-pc-windows-msvc
CHANNEL: stable
BITS: 32
matrix:
fast_finish: true
# Install Rust and Cargo
# (Based on from https://github.com/rust-lang/libc/blob/master/appveyor.yml)
install:
- curl -sSf -o rustup-init.exe https://win.rustup.rs/
- rustup-init.exe -y --default-host %TARGET%
- set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
- if defined MSYS2 set PATH=C:\msys64\mingw%BITS%\bin;%PATH%
- rustc -V
- cargo -V
# Hack to work around a harmless warning in Appveyor builds?
build: false
# Equivalent to Travis' `script` phase
test_script:
- cargo test --verbose --all --features pcre2
before_deploy:
# Generate artifacts for release
- cargo build --release --features pcre2
- mkdir staging
- copy target\release\rg.exe staging
- ps: copy target\release\build\ripgrep-*\out\_rg.ps1 staging
- cd staging
# release zipfile will look like 'ripgrep-1.2.3-x86_64-pc-windows-msvc'
- 7z a ../%PROJECT_NAME%-%APPVEYOR_REPO_TAG_NAME%-%TARGET%.zip *
- appveyor PushArtifact ../%PROJECT_NAME%-%APPVEYOR_REPO_TAG_NAME%-%TARGET%.zip
deploy:
description: 'Automatically deployed release'
# All the zipped artifacts will be deployed
artifact: /.*\.zip/
auth_token:
secure: vv4vBCEosGlyQjaEC1+kraP2P6O4CQSa+Tw50oHWFTGcmuXxaWS0/yEXbxsIRLpw
provider: GitHub
# deploy when a new tag is pushed and only on the stable channel
on:
CHANNEL: stable
APPVEYOR_REPO_TAG: true
branches:
only:
- /^\d+\.\d+\.\d+$/
- master

81
build.rs
View File

@@ -1,8 +1,3 @@
#[macro_use]
extern crate clap;
#[macro_use]
extern crate lazy_static;
use std::env;
use std::fs::{self, File};
use std::io::{self, Read, Write};
@@ -14,7 +9,7 @@ use clap::Shell;
use app::{RGArg, RGArgKind};
#[allow(dead_code)]
#[path = "src/app.rs"]
#[path = "crates/core/app.rs"]
mod app;
fn main() {
@@ -26,7 +21,8 @@ fn main() {
eprintln!(
"OUT_DIR environment variable not defined. \
Please file a bug: \
https://github.com/BurntSushi/ripgrep/issues/new");
https://github.com/BurntSushi/ripgrep/issues/new"
);
process::exit(1);
}
};
@@ -69,6 +65,51 @@ fn git_revision_hash() -> Option<String> {
}
fn generate_man_page<P: AsRef<Path>>(outdir: P) -> io::Result<()> {
// If asciidoctor isn't installed, fallback to asciidoc.
if let Err(err) = process::Command::new("asciidoctor").output() {
eprintln!(
"Could not run 'asciidoctor' binary, falling back to 'a2x'."
);
eprintln!("Error from running 'asciidoctor': {}", err);
return legacy_generate_man_page::<P>(outdir);
}
// 1. Read asciidoctor template.
// 2. Interpolate template with auto-generated docs.
// 3. Save interpolation to disk.
// 4. Use asciidoctor to convert to man page.
let outdir = outdir.as_ref();
let cwd = env::current_dir()?;
let tpl_path = cwd.join("doc").join("rg.1.txt.tpl");
let txt_path = outdir.join("rg.1.txt");
let mut tpl = String::new();
File::open(&tpl_path)?.read_to_string(&mut tpl)?;
let options =
formatted_options()?.replace("&#123;", "{").replace("&#125;", "}");
tpl = tpl.replace("{OPTIONS}", &options);
let githash = git_revision_hash();
let githash = githash.as_ref().map(|x| &**x);
tpl = tpl.replace("{VERSION}", &app::long_version(githash, false));
File::create(&txt_path)?.write_all(tpl.as_bytes())?;
let result = process::Command::new("asciidoctor")
.arg("--doctype")
.arg("manpage")
.arg("--backend")
.arg("manpage")
.arg(&txt_path)
.spawn()?
.wait()?;
if !result.success() {
let msg =
format!("'asciidoctor' failed with exit code {:?}", result.code());
return Err(ioerr(msg));
}
Ok(())
}
fn legacy_generate_man_page<P: AsRef<Path>>(outdir: P) -> io::Result<()> {
// If asciidoc isn't installed, then don't do anything.
if let Err(err) = process::Command::new("a2x").output() {
eprintln!("Could not run 'a2x' binary, skipping man page generation.");
@@ -90,13 +131,15 @@ fn generate_man_page<P: AsRef<Path>>(outdir: P) -> io::Result<()> {
let githash = git_revision_hash();
let githash = githash.as_ref().map(|x| &**x);
tpl = tpl.replace("{VERSION}", &app::long_version(githash));
tpl = tpl.replace("{VERSION}", &app::long_version(githash, false));
File::create(&txt_path)?.write_all(tpl.as_bytes())?;
let result = process::Command::new("a2x")
.arg("--no-xmllint")
.arg("--doctype").arg("manpage")
.arg("--format").arg("manpage")
.arg("--doctype")
.arg("manpage")
.arg("--format")
.arg("manpage")
.arg(&txt_path)
.spawn()?
.wait()?;
@@ -119,7 +162,7 @@ fn formatted_options() -> io::Result<String> {
// ripgrep only has two positional arguments, and probably will only
// ever have two positional arguments, so we just hardcode them into
// the template.
if let app::RGArgKind::Positional{..} = arg.kind {
if let app::RGArgKind::Positional { .. } = arg.kind {
continue;
}
formatted.push(formatted_arg(&arg)?);
@@ -129,7 +172,9 @@ fn formatted_options() -> io::Result<String> {
fn formatted_arg(arg: &RGArg) -> io::Result<String> {
match arg.kind {
RGArgKind::Positional{..} => panic!("unexpected positional argument"),
RGArgKind::Positional { .. } => {
panic!("unexpected positional argument")
}
RGArgKind::Switch { long, short, multiple } => {
let mut out = vec![];
@@ -168,7 +213,17 @@ fn formatted_arg(arg: &RGArg) -> io::Result<String> {
}
fn formatted_doc_txt(arg: &RGArg) -> io::Result<String> {
let paragraphs: Vec<&str> = arg.doc_long.split("\n\n").collect();
let paragraphs: Vec<String> = arg
.doc_long
.replace("{", "&#123;")
.replace("}", r"&#125;")
// Hack to render ** literally in man page correctly. We can't put
// these crazy +++ in the help text directly, since that shows
// literally in --help output.
.replace("*-g 'foo/**'*", "*-g +++'foo/**'+++*")
.split("\n\n")
.map(|s| s.to_string())
.collect();
if paragraphs.is_empty() {
return Err(ioerr(format!("missing docs for --{}", arg.name)));
}

61
ci/before_deploy.sh
View File

@@ -1,61 +0,0 @@
#!/bin/bash
# package the build artifacts
set -ex
. "$(dirname $0)/utils.sh"
# Generate artifacts for release
mk_artifacts() {
if is_arm; then
cargo build --target "$TARGET" --release
else
# Technically, MUSL builds will force PCRE2 to get statically compiled,
# but we also want PCRE2 statically build for macOS binaries.
PCRE2_SYS_STATIC=1 cargo build --target "$TARGET" --release --features 'pcre2'
fi
}
mk_tarball() {
# When cross-compiling, use the right `strip` tool on the binary.
local gcc_prefix="$(gcc_prefix)"
# Create a temporary dir that contains our staging area.
# $tmpdir/$name is what eventually ends up as the deployed archive.
local tmpdir="$(mktemp -d)"
local name="${PROJECT_NAME}-${TRAVIS_TAG}-${TARGET}"
local staging="$tmpdir/$name"
mkdir -p "$staging"/{complete,doc}
# The deployment directory is where the final archive will reside.
# This path is known by the .travis.yml configuration.
local out_dir="$(pwd)/deployment"
mkdir -p "$out_dir"
# Find the correct (most recent) Cargo "out" directory. The out directory
# contains shell completion files and the man page.
local cargo_out_dir="$(cargo_out_dir "target/$TARGET")"
# Copy the ripgrep binary and strip it.
cp "target/$TARGET/release/rg" "$staging/rg"
"${gcc_prefix}strip" "$staging/rg"
# Copy the licenses and README.
cp {README.md,UNLICENSE,COPYING,LICENSE-MIT} "$staging/"
# Copy documentation and man page.
cp {CHANGELOG.md,FAQ.md,GUIDE.md} "$staging/doc/"
if command -V a2x 2>&1 > /dev/null; then
# The man page should only exist if we have asciidoc installed.
cp "$cargo_out_dir/rg.1" "$staging/doc/"
fi
# Copy shell completion files.
cp "$cargo_out_dir"/{rg.bash,rg.fish,_rg.ps1} "$staging/complete/"
cp complete/_rg "$staging/complete/"
(cd "$tmpdir" && tar czf "$out_dir/$name.tar.gz" "$name")
rm -rf "$tmpdir"
}
main() {
mk_artifacts
mk_tarball
}
main

16
ci/build_deb.sh → ci/build-deb
View File

@@ -1,6 +1,7 @@
#!/bin/bash
set -e
D="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
# This script builds a binary dpkg for Debian based distros. It does not
# currently run in CI, and is instead run manually and the resulting dpkg is
@@ -23,20 +24,13 @@ fi
# the deb, which knows where to look.
DEPLOY_DIR=deployment/deb
OUT_DIR="$("$D"/cargo-out-dir target/debug/)"
mkdir -p "$DEPLOY_DIR"
cargo build
# Find and copy man page.
manpage="$(find ./target/debug -name rg.1 -print0 | xargs -0 ls -t | head -n1)"
cp "$manpage" "$DEPLOY_DIR/"
# Do the same for shell completions.
compbash="$(find ./target/debug -name rg.bash -print0 | xargs -0 ls -t | head -n1)"
cp "$compbash" "$DEPLOY_DIR/"
compfish="$(find ./target/debug -name rg.fish -print0 | xargs -0 ls -t | head -n1)"
cp "$compfish" "$DEPLOY_DIR/"
compzsh="complete/_rg"
cp "$compzsh" "$DEPLOY_DIR/"
# Copy man page and shell completions.
cp "$OUT_DIR"/{rg.1,rg.bash,rg.fish} "$DEPLOY_DIR/"
cp complete/_rg "$DEPLOY_DIR/"
# Since we're distributing the dpkg, we don't know whether the user will have
# PCRE2 installed, so just do a static build.

19
ci/cargo-out-dir Executable file
View File

@@ -0,0 +1,19 @@
#!/bin/bash
# Finds Cargo's `OUT_DIR` directory from the most recent build.
#
# This requires one parameter corresponding to the target directory
# to search for the build output.
if [ $# != 1 ]; then
echo "Usage: $(basename "$0") <target-dir>" >&2
exit 2
fi
# This works by finding the most recent stamp file, which is produced by
# every ripgrep build.
target_dir="$1"
find "$target_dir" -name ripgrep-stamp -print0 \
| xargs -0 ls -t \
| head -n1 \
| xargs dirname

24
ci/docker/README.md Normal file
View File

@@ -0,0 +1,24 @@
These are Docker images used for cross compilation in CI builds (or locally)
via the [Cross](https://github.com/rust-embedded/cross) tool.
The Cross tool actually provides its own Docker images, and all Docker images
in this directory are derived from one of them. We provide our own in order
to customize the environment. For example, we need to install some things like
`asciidoctor` in order to generate man pages. We also install compression tools
like `xz` so that tests for the `-z/--search-zip` flag are run.
If you make a change to a Docker image, then you can re-build it. `cd` into the
directory containing the `Dockerfile` and run:
$ cd x86_64-unknown-linux-musl
$ ./build
At this point, subsequent uses of `cross` will now use your built image since
Docker prefers local images over remote images. In order to make these changes
stick, they need to be pushed to Docker Hub:
$ docker push burntsushi/cross:x86_64-unknown-linux-musl
Of course, only I (BurntSushi) can push to that location. To make `cross` use
a different location, then edit `Cross.toml` in the root of this repo to use
a different image name for the desired target.

4
ci/docker/arm-unknown-linux-gnueabihf/Dockerfile Normal file
View File

@@ -0,0 +1,4 @@
FROM rustembedded/cross:arm-unknown-linux-gnueabihf
COPY stage/ubuntu-install-packages /
RUN /ubuntu-install-packages

5
ci/docker/arm-unknown-linux-gnueabihf/build Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
mkdir -p stage
cp ../../ubuntu-install-packages ./stage/
docker build -t burntsushi/cross:arm-unknown-linux-gnueabihf .

4
ci/docker/i686-unknown-linux-gnu/Dockerfile Normal file
View File

@@ -0,0 +1,4 @@
FROM rustembedded/cross:i686-unknown-linux-gnu
COPY stage/ubuntu-install-packages /
RUN /ubuntu-install-packages

5
ci/docker/i686-unknown-linux-gnu/build Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
mkdir -p stage
cp ../../ubuntu-install-packages ./stage/
docker build -t burntsushi/cross:i686-unknown-linux-gnu .

4
ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile Normal file
View File

@@ -0,0 +1,4 @@
FROM rustembedded/cross:mips64-unknown-linux-gnuabi64
COPY stage/ubuntu-install-packages /
RUN /ubuntu-install-packages

5
ci/docker/mips64-unknown-linux-gnuabi64/build Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
mkdir -p stage
cp ../../ubuntu-install-packages ./stage/
docker build -t burntsushi/cross:mips64-unknown-linux-gnuabi64 .

4
ci/docker/x86_64-unknown-linux-musl/Dockerfile Normal file
View File

@@ -0,0 +1,4 @@
FROM rustembedded/cross:x86_64-unknown-linux-musl
COPY stage/ubuntu-install-packages /
RUN /ubuntu-install-packages

5
ci/docker/x86_64-unknown-linux-musl/build Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
mkdir -p stage
cp ../../ubuntu-install-packages ./stage/
docker build -t burntsushi/cross:x86_64-unknown-linux-musl .

61
ci/install.sh
View File

@@ -1,61 +0,0 @@
#!/bin/bash
# install stuff needed for the `script` phase
# Where rustup gets installed.
export PATH="$PATH:$HOME/.cargo/bin"
set -ex
. "$(dirname $0)/utils.sh"
install_rustup() {
curl https://sh.rustup.rs -sSf \
| sh -s -- -y --default-toolchain="$TRAVIS_RUST_VERSION"
rustc -V
cargo -V
}
install_targets() {
if [ $(host) != "$TARGET" ]; then
rustup target add $TARGET
fi
}
install_osx_dependencies() {
if ! is_osx; then
return
fi
brew install asciidoc docbook-xsl
}
configure_cargo() {
local prefix=$(gcc_prefix)
if [ -n "${prefix}" ]; then
local gcc_suffix=
if [ -n "$GCC_VERSION" ]; then
gcc_suffix="-$GCC_VERSION"
fi
local gcc="${prefix}gcc${gcc_suffix}"
# information about the cross compiler
"${gcc}" -v
# tell cargo which linker to use for cross compilation
mkdir -p .cargo
cat >>.cargo/config <<EOF
[target.$TARGET]
linker = "${gcc}"
EOF
fi
}
main() {
install_osx_dependencies
install_rustup
install_targets
configure_cargo
}
main

3
ci/macos-install-packages Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/sh
brew install asciidoctor

50
ci/script.sh
View File

@@ -1,50 +0,0 @@
#!/bin/bash
# build, test and generate docs in this phase
set -ex
. "$(dirname $0)/utils.sh"
main() {
# Test a normal debug build.
if is_arm; then
cargo build --target "$TARGET" --verbose
else
cargo build --target "$TARGET" --verbose --all --features 'pcre2'
fi
# Show the output of the most recent build.rs stderr.
set +x
stderr="$(find "target/$TARGET/debug" -name stderr -print0 | xargs -0 ls -t | head -n1)"
if [ -s "$stderr" ]; then
echo "===== $stderr ====="
cat "$stderr"
echo "====="
fi
set -x
# sanity check the file type
file target/"$TARGET"/debug/rg
# Check that we've generated man page and other shell completions.
outdir="$(cargo_out_dir "target/$TARGET/debug")"
file "$outdir/rg.bash"
file "$outdir/rg.fish"
file "$outdir/_rg.ps1"
file "$outdir/rg.1"
# Apparently tests don't work on arm, so just bail now. I guess we provide
# ARM releases on a best effort basis?
if is_arm; then
return 0
fi
# Test that zsh completions are in sync with ripgrep's actual args.
"$(dirname "${0}")/test_complete.sh"
# Run tests for ripgrep and all sub-crates.
cargo test --target "$TARGET" --verbose --all --features 'pcre2'
}
main

0
ci/sha256.sh → ci/sha256-releases
View File

4
ci/test_complete.sh → ci/test-complete
View File

@@ -18,7 +18,7 @@ get_comp_args() {
main() {
local diff
local rg="${0:a:h}/../target/${TARGET:-}/release/rg"
local rg="${0:a:h}/../${TARGET_DIR:-target}/release/rg"
local _rg="${0:a:h}/../complete/_rg"
local -a help_args comp_args
@@ -44,7 +44,7 @@ main() {
# Occasionally we may have to handle some manually, however
help_args=( ${(f)"$(
$rg --help |
$rg -i -- '^\s+--?[a-z0-9]|--[imnp]' |
$rg -i -- '^\s+--?[a-z0-9]|--[a-z]' |
$rg -ior '$1' -- $'[\t /\"\'`.,](-[a-z0-9]|--[a-z0-9-]+)\\b' |
$rg -v -- --print0 | # False positives
sort -u

6
ci/ubuntu-install-packages Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/sh
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
asciidoctor \
zsh xz-utils liblz4-tool musl-tools

25
ci/utils.sh
View File

@@ -55,6 +55,13 @@ gcc_prefix() {
esac
}
is_musl() {
case "$TARGET" in
*-musl) return 0 ;;
*) return 1 ;;
esac
}
is_x86() {
case "$(architecture)" in
amd64|i386) return 0 ;;
@@ -62,6 +69,13 @@ is_x86() {
esac
}
is_x86_64() {
case "$(architecture)" in
amd64) return 0 ;;
*) return 1 ;;
esac
}
is_arm() {
case "$(architecture)" in
armhf) return 0 ;;
@@ -82,3 +96,14 @@ is_osx() {
*) return 1 ;;
esac
}
builder() {
if is_musl && is_x86_64; then
# cargo install cross
# To work around https://github.com/rust-embedded/cross/issues/357
cargo install --git https://github.com/rust-embedded/cross --force
echo "cross"
else
echo "cargo"
fi
}

61
complete/_rg
View File

@@ -3,7 +3,7 @@
##
# zsh completion function for ripgrep
#
# Run ci/test_complete.sh after building to ensure that the options supported by
# Run ci/test-complete after building to ensure that the options supported by
# this function stay in synch with the `rg` binary.
#
# For convenience, a completion reference guide is included at the bottom of
@@ -43,6 +43,7 @@ _rg() {
+ '(exclusive)' # Misc. fully exclusive options
'(: * -)'{-h,--help}'[display help information]'
'(: * -)'{-V,--version}'[display version information]'
'(: * -)'--pcre2-version'[print the version of PCRE2 used by ripgrep, if available]'
+ '(buffered)' # buffering options
'--line-buffered[force line buffering]'
@@ -71,11 +72,19 @@ _rg() {
+ '(count)' # Counting options
{-c,--count}'[only show count of matching lines for each file]'
'--count-matches[only show count of individual matches for each file]'
'--include-zero[include files with zero matches in summary]'
+ '(encoding)' # Encoding options
{-E+,--encoding=}'[specify text encoding of files to search]: :_rg_encodings'
$no'--no-encoding[use default text encoding]'
+ '(engine)' # Engine choice options
'--engine=[select which regex engine to use]:when:((
default\:"use default engine"
pcre2\:"identical to --pcre2"
auto\:"identical to --auto-hybrid-regex"
))'
+ file # File-input options
'(1)*'{-f+,--file=}'[specify file containing patterns to search for]: :_files'
@@ -85,7 +94,7 @@ _rg() {
+ '(file-name)' # File-name options
{-H,--with-filename}'[show file name for matches]'
"--no-filename[don't show file name for matches]"
{-I,--no-filename}"[don't show file name for matches]"
+ '(file-system)' # File system options
"--one-file-system[don't descend into directories on other file systems]"
@@ -103,6 +112,10 @@ _rg() {
'*'{-g+,--glob=}'[include/exclude files matching specified glob]:glob'
'*--iglob=[include/exclude files matching specified case-insensitive glob]:glob'
+ '(glob-case-insensitive)' # File-glob case sensitivity options
'--glob-case-insensitive[treat -g/--glob patterns case insensitively]'
$no'--no-glob-case-insensitive[treat -g/--glob patterns case sensitively]'
+ '(heading)' # Heading options
'(pretty-vimgrep)--heading[show matches grouped by file name]'
"(pretty-vimgrep)--no-heading[don't show matches grouped by file name]"
@@ -111,9 +124,21 @@ _rg() {
'--hidden[search hidden files and directories]'
$no"--no-hidden[don't search hidden files and directories]"
+ '(hybrid)' # hybrid regex options
'--auto-hybrid-regex[dynamically use PCRE2 if necessary]'
$no"--no-auto-hybrid-regex[don't dynamically use PCRE2 if necessary]"
+ '(ignore)' # Ignore-file options
"(--no-ignore-global --no-ignore-parent --no-ignore-vcs)--no-ignore[don't respect ignore files]"
$no'(--ignore-global --ignore-parent --ignore-vcs)--ignore[respect ignore files]'
"(--no-ignore-global --no-ignore-parent --no-ignore-vcs --no-ignore-dot)--no-ignore[don't respect ignore files]"
$no'(--ignore-global --ignore-parent --ignore-vcs --ignore-dot)--ignore[respect ignore files]'
+ '(ignore-file-case-insensitive)' # Ignore-file case sensitivity options
'--ignore-file-case-insensitive[process ignore files case insensitively]'
$no'--no-ignore-file-case-insensitive[process ignore files case sensitively]'
+ '(ignore-exclude)' # Local exclude (ignore)-file options
"--no-ignore-exclude[don't respect local exclude (ignore) files]"
$no'--ignore-exclude[respect local exclude (ignore) files]'
+ '(ignore-global)' # Global ignore-file options
"--no-ignore-global[don't respect global ignore files]"
@@ -127,6 +152,18 @@ _rg() {
"--no-ignore-vcs[don't respect version control ignore files]"
$no'--ignore-vcs[respect version control ignore files]'
+ '(require-git)' # git specific settings
"--no-require-git[don't require git repository to respect gitignore rules]"
$no'--require-git[require git repository to respect gitignore rules]'
+ '(ignore-dot)' # .ignore options
"--no-ignore-dot[don't respect .ignore files]"
$no'--ignore-dot[respect .ignore files]'
+ '(ignore-files)' # custom global ignore file options
"--no-ignore-files[don't respect --ignore-file flags]"
$no'--ignore-files[respect --ignore-file files]'
+ '(json)' # JSON options
'--json[output results in JSON Lines format]'
$no"--no-json[don't output results in JSON Lines format]"
@@ -140,6 +177,10 @@ _rg() {
$no"--no-crlf[don't use CRLF as line terminator]"
'(text)--null-data[use NUL as line terminator]'
+ '(max-columns-preview)' # max column preview options
'--max-columns-preview[show preview for long lines (with -M)]'
$no"--no-max-columns-preview[don't show preview for long lines (with -M)]"
+ '(max-depth)' # Directory-depth options
'--max-depth=[specify max number of directories to descend]:number of directories'
'!--maxdepth=:number of directories'
@@ -219,6 +260,8 @@ _rg() {
+ '(text)' # Binary-search options
{-a,--text}'[search binary files as if they were text]'
"--binary[search binary files, don't print binary data]"
$no"--no-binary[don't search binary files]"
$no"(--null-data)--no-text[don't search binary files as if they were text]"
+ '(threads)' # Thread-count options
@@ -240,6 +283,10 @@ _rg() {
{-w,--word-regexp}'[only show matches surrounded by word boundaries]'
{-x,--line-regexp}'[only show matches surrounded by line boundaries]'
+ '(unicode)' # Unicode options
$no'--unicode[enable Unicode mode]'
'--no-unicode[disable Unicode mode]'
+ '(zip)' # Compression options
'(--pre)'{-z,--search-zip}'[search in compressed files]'
$no"--no-search-zip[don't search in compressed files]"
@@ -254,7 +301,9 @@ _rg() {
))'
'*--colors=[specify color and style settings]: :->colorspec'
'--context-separator=[specify string used to separate non-continuous context lines in output]:separator'
$no"--no-context-separator[don't print context separators]"
'--debug[show debug messages]'
'--trace[show more verbose debug messages]'
'--dfa-size-limit=[specify upper size limit of generated DFA]:DFA size (bytes)'
"(1 stats)--files[show each file that would be searched (but don't search)]"
'*--ignore-file=[specify additional ignore file]:ignore file:_files'
@@ -274,7 +323,7 @@ _rg() {
'(--type-list)*: :_files'
)
# This is used with test_complete.sh to verify that there are no options
# This is used with test-complete to verify that there are no options
# listed in the help output that aren't also defined here
[[ $_RG_COMPLETE_LIST_ARGS == (1|t*|y*) ]] && {
print -rl - $args
@@ -370,7 +419,7 @@ _rg_encodings() {
shift{-,_}jis csshiftjis {,x-}sjis ms_kanji ms932
utf{,-}8 utf-16{,be,le} unicode-1-1-utf-8
windows-{31j,874,949,125{0..8}} dos-874 tis-620 ansi_x3.4-1968
x-user-defined auto
x-user-defined auto none
)
_wanted encodings expl encoding compadd -a "$@" - _encodings

15
grep-cli/Cargo.toml → crates/cli/Cargo.toml
View File

@@ -1,25 +1,26 @@
[package]
name = "grep-cli"
version = "0.1.1" #:version
version = "0.1.5" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """
Utilities for search oriented command line applications.
"""
documentation = "https://docs.rs/grep-cli"
homepage = "https://github.com/BurntSushi/ripgrep"
repository = "https://github.com/BurntSushi/ripgrep"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/cli"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/cli"
readme = "README.md"
keywords = ["regex", "grep", "cli", "utility", "util"]
license = "Unlicense/MIT"
[dependencies]
atty = "0.2.11"
globset = { version = "0.4.2", path = "../globset" }
bstr = "0.2.0"
globset = { version = "0.4.5", path = "../globset" }
lazy_static = "1.1.0"
log = "0.4.5"
regex = "1.0.5"
same-file = "1.0.3"
termcolor = "1.0.3"
regex = "1.1"
same-file = "1.0.4"
termcolor = "1.0.4"
[target.'cfg(windows)'.dependencies.winapi-util]
version = "0.1.1"

0
globset/LICENSE-MIT → crates/cli/LICENSE-MIT
View File

0
grep-cli/README.md → crates/cli/README.md
View File

0
globset/UNLICENSE → crates/cli/UNLICENSE
View File

41
grep-cli/src/decompress.rs → crates/cli/src/decompress.rs
View File

@@ -38,10 +38,7 @@ impl Default for DecompressionMatcherBuilder {
impl DecompressionMatcherBuilder {
/// Create a new builder for configuring a decompression matcher.
pub fn new() -> DecompressionMatcherBuilder {
DecompressionMatcherBuilder {
commands: vec![],
defaults: true,
}
DecompressionMatcherBuilder { commands: vec![], defaults: true }
}
/// Build a matcher for determining how to decompress files.
@@ -49,12 +46,11 @@ impl DecompressionMatcherBuilder {
/// If there was a problem compiling the matcher, then an error is
/// returned.
pub fn build(&self) -> Result<DecompressionMatcher, CommandError> {
let defaults =
if !self.defaults {
vec![]
} else {
default_decompression_commands()
};
let defaults = if !self.defaults {
vec![]
} else {
default_decompression_commands()
};
let mut glob_builder = GlobSetBuilder::new();
let mut commands = vec![];
for decomp_cmd in defaults.iter().chain(&self.commands) {
@@ -93,17 +89,15 @@ impl DecompressionMatcherBuilder {
program: P,
args: I,
) -> &mut DecompressionMatcherBuilder
where P: AsRef<OsStr>,
I: IntoIterator<Item=A>,
A: AsRef<OsStr>,
where
P: AsRef<OsStr>,
I: IntoIterator<Item = A>,
A: AsRef<OsStr>,
{
let glob = glob.to_string();
let bin = program.as_ref().to_os_string();
let args = args
.into_iter()
.map(|a| a.as_ref().to_os_string())
.collect();
let args =
args.into_iter().map(|a| a.as_ref().to_os_string()).collect();
self.commands.push(DecompressionCommand { glob, bin, args });
self
}
@@ -352,6 +346,9 @@ fn default_decompression_commands() -> Vec<DecompressionCommand> {
const ARGS_XZ: &[&str] = &["xz", "-d", "-c"];
const ARGS_LZ4: &[&str] = &["lz4", "-d", "-c"];
const ARGS_LZMA: &[&str] = &["xz", "--format=lzma", "-d", "-c"];
const ARGS_BROTLI: &[&str] = &["brotli", "-d", "-c"];
const ARGS_ZSTD: &[&str] = &["zstd", "-q", "-d", "-c"];
const ARGS_UNCOMPRESS: &[&str] = &["uncompress", "-c"];
fn cmd(glob: &str, args: &[&str]) -> DecompressionCommand {
DecompressionCommand {
@@ -367,15 +364,15 @@ fn default_decompression_commands() -> Vec<DecompressionCommand> {
vec![
cmd("*.gz", ARGS_GZIP),
cmd("*.tgz", ARGS_GZIP),
cmd("*.bz2", ARGS_BZIP),
cmd("*.tbz2", ARGS_BZIP),
cmd("*.xz", ARGS_XZ),
cmd("*.txz", ARGS_XZ),
cmd("*.lz4", ARGS_LZ4),
cmd("*.lzma", ARGS_LZMA),
cmd("*.br", ARGS_BROTLI),
cmd("*.zst", ARGS_ZSTD),
cmd("*.zstd", ARGS_ZSTD),
cmd("*.Z", ARGS_UNCOMPRESS),
]
}

167
grep-cli/src/escape.rs → crates/cli/src/escape.rs
View File

@@ -1,6 +1,8 @@
use std::ffi::OsStr;
use std::str;
use bstr::{ByteSlice, ByteVec};
/// A single state in the state machine used by `unescape`.
#[derive(Clone, Copy, Eq, PartialEq)]
enum State {
@@ -35,18 +37,15 @@ enum State {
///
/// assert_eq!(r"foo\nbar\xFFbaz", escape(b"foo\nbar\xFFbaz"));
/// ```
pub fn escape(mut bytes: &[u8]) -> String {
pub fn escape(bytes: &[u8]) -> String {
let mut escaped = String::new();
while let Some(result) = decode_utf8(bytes) {
match result {
Ok(cp) => {
escape_char(cp, &mut escaped);
bytes = &bytes[cp.len_utf8()..];
}
Err(byte) => {
escape_byte(byte, &mut escaped);
bytes = &bytes[1..];
for (s, e, ch) in bytes.char_indices() {
if ch == '\u{FFFD}' {
for b in bytes[s..e].bytes() {
escape_byte(b, &mut escaped);
}
} else {
escape_char(ch, &mut escaped);
}
}
escaped
@@ -56,19 +55,7 @@ pub fn escape(mut bytes: &[u8]) -> String {
///
/// This is like [`escape`](fn.escape.html), but accepts an OS string.
pub fn escape_os(string: &OsStr) -> String {
#[cfg(unix)]
fn imp(string: &OsStr) -> String {
use std::os::unix::ffi::OsStrExt;
escape(string.as_bytes())
}
#[cfg(not(unix))]
fn imp(string: &OsStr) -> String {
escape(string.to_string_lossy().as_bytes())
}
imp(string)
escape(Vec::from_os_str_lossy(string).as_bytes())
}
/// Unescapes a string.
@@ -108,51 +95,61 @@ pub fn unescape(s: &str) -> Vec<u8> {
let mut state = Literal;
for c in s.chars() {
match state {
Escape => {
match c {
'\\' => { bytes.push(b'\\'); state = Literal; }
'n' => { bytes.push(b'\n'); state = Literal; }
'r' => { bytes.push(b'\r'); state = Literal; }
't' => { bytes.push(b'\t'); state = Literal; }
'x' => { state = HexFirst; }
c => {
bytes.extend(format!(r"\{}", c).into_bytes());
state = Literal;
}
Escape => match c {
'\\' => {
bytes.push(b'\\');
state = Literal;
}
}
HexFirst => {
match c {
'0'...'9' | 'A'...'F' | 'a'...'f' => {
state = HexSecond(c);
}
c => {
bytes.extend(format!(r"\x{}", c).into_bytes());
state = Literal;
}
'n' => {
bytes.push(b'\n');
state = Literal;
}
}
HexSecond(first) => {
match c {
'0'...'9' | 'A'...'F' | 'a'...'f' => {
let ordinal = format!("{}{}", first, c);
let byte = u8::from_str_radix(&ordinal, 16).unwrap();
bytes.push(byte);
state = Literal;
}
c => {
let original = format!(r"\x{}{}", first, c);
bytes.extend(original.into_bytes());
state = Literal;
}
'r' => {
bytes.push(b'\r');
state = Literal;
}
}
Literal => {
match c {
'\\' => { state = Escape; }
c => { bytes.extend(c.to_string().as_bytes()); }
't' => {
bytes.push(b'\t');
state = Literal;
}
}
'x' => {
state = HexFirst;
}
c => {
bytes.extend(format!(r"\{}", c).into_bytes());
state = Literal;
}
},
HexFirst => match c {
'0'..='9' | 'A'..='F' | 'a'..='f' => {
state = HexSecond(c);
}
c => {
bytes.extend(format!(r"\x{}", c).into_bytes());
state = Literal;
}
},
HexSecond(first) => match c {
'0'..='9' | 'A'..='F' | 'a'..='f' => {
let ordinal = format!("{}{}", first, c);
let byte = u8::from_str_radix(&ordinal, 16).unwrap();
bytes.push(byte);
state = Literal;
}
c => {
let original = format!(r"\x{}{}", first, c);
bytes.extend(original.into_bytes());
state = Literal;
}
},
Literal => match c {
'\\' => {
state = Escape;
}
c => {
bytes.extend(c.to_string().as_bytes());
}
},
}
}
match state {
@@ -186,7 +183,7 @@ fn escape_char(cp: char, into: &mut String) {
/// Adds the given byte to the given string, escaping it if necessary.
fn escape_byte(byte: u8, into: &mut String) {
match byte {
0x21...0x5B | 0x5D...0x7D => into.push(byte as char),
0x21..=0x5B | 0x5D..=0x7D => into.push(byte as char),
b'\n' => into.push_str(r"\n"),
b'\r' => into.push_str(r"\r"),
b'\t' => into.push_str(r"\t"),
@@ -195,46 +192,6 @@ fn escape_byte(byte: u8, into: &mut String) {
}
}
/// Decodes the next UTF-8 encoded codepoint from the given byte slice.
///
/// If no valid encoding of a codepoint exists at the beginning of the given
/// byte slice, then the first byte is returned instead.
///
/// This returns `None` if and only if `bytes` is empty.
fn decode_utf8(bytes: &[u8]) -> Option<Result<char, u8>> {
if bytes.is_empty() {
return None;
}
let len = match utf8_len(bytes[0]) {
None => return Some(Err(bytes[0])),
Some(len) if len > bytes.len() => return Some(Err(bytes[0])),
Some(len) => len,
};
match str::from_utf8(&bytes[..len]) {
Ok(s) => Some(Ok(s.chars().next().unwrap())),
Err(_) => Some(Err(bytes[0])),
}
}
/// Given a UTF-8 leading byte, this returns the total number of code units
/// in the following encoded codepoint.
///
/// If the given byte is not a valid UTF-8 leading byte, then this returns
/// `None`.
fn utf8_len(byte: u8) -> Option<usize> {
if byte <= 0x7F {
Some(1)
} else if byte <= 0b110_11111 {
Some(2)
} else if byte <= 0b1110_1111 {
Some(3)
} else if byte <= 0b1111_0111 {
Some(4)
} else {
None
}
}
#[cfg(test)]
mod tests {
use super::{escape, unescape};

50
grep-cli/src/human.rs → crates/cli/src/human.rs
View File

@@ -46,7 +46,9 @@ impl ParseSizeError {
}
impl error::Error for ParseSizeError {
fn description(&self) -> &str { "invalid size" }
fn description(&self) -> &str {
"invalid size"
}
}
impl fmt::Display for ParseSizeError {
@@ -54,26 +56,19 @@ impl fmt::Display for ParseSizeError {
use self::ParseSizeErrorKind::*;
match self.kind {
InvalidFormat => {
write!(
f,
"invalid format for size '{}', which should be a sequence \
InvalidFormat => write!(
f,
"invalid format for size '{}', which should be a sequence \
of digits followed by an optional 'K', 'M' or 'G' \
suffix",
self.original
)
}
InvalidInt(ref err) => {
write!(
f,
"invalid integer found in size '{}': {}",
self.original,
err
)
}
Overflow => {
write!(f, "size too big in '{}'", self.original)
}
self.original
),
InvalidInt(ref err) => write!(
f,
"invalid integer found in size '{}': {}",
self.original, err
),
Overflow => write!(f, "size too big in '{}'", self.original),
}
}
}
@@ -104,17 +99,16 @@ pub fn parse_human_readable_size(size: &str) -> Result<u64, ParseSizeError> {
Some(caps) => caps,
None => return Err(ParseSizeError::format(size)),
};
let value: u64 = caps[1].parse().map_err(|err| {
ParseSizeError::int(size, err)
})?;
let value: u64 =
caps[1].parse().map_err(|err| ParseSizeError::int(size, err))?;
let suffix = match caps.get(2) {
None => return Ok(value),
Some(cap) => cap.as_str(),
};
let bytes = match suffix {
"K" => value.checked_mul(1<<10),
"M" => value.checked_mul(1<<20),
"G" => value.checked_mul(1<<30),
"K" => value.checked_mul(1 << 10),
"M" => value.checked_mul(1 << 20),
"G" => value.checked_mul(1 << 30),
// Because if the regex matches this group, it must be [KMG].
_ => unreachable!(),
};
@@ -134,19 +128,19 @@ mod tests {
#[test]
fn suffix_k() {
let x = parse_human_readable_size("123K").unwrap();
assert_eq!(123 * (1<<10), x);
assert_eq!(123 * (1 << 10), x);
}
#[test]
fn suffix_m() {
let x = parse_human_readable_size("123M").unwrap();
assert_eq!(123 * (1<<20), x);
assert_eq!(123 * (1 << 20), x);
}
#[test]
fn suffix_g() {
let x = parse_human_readable_size("123G").unwrap();
assert_eq!(123 * (1<<30), x);
assert_eq!(123 * (1 << 30), x);
}
#[test]

17
grep-cli/src/lib.rs → crates/cli/src/lib.rs
View File

@@ -159,6 +159,7 @@ error message is crafted that typically tells the user how to fix the problem.
#![deny(missing_docs)]
extern crate atty;
extern crate bstr;
extern crate globset;
#[macro_use]
extern crate lazy_static;
@@ -178,20 +179,18 @@ mod process;
mod wtr;
pub use decompress::{
DecompressionMatcher, DecompressionMatcherBuilder,
DecompressionReader, DecompressionReaderBuilder,
DecompressionMatcher, DecompressionMatcherBuilder, DecompressionReader,
DecompressionReaderBuilder,
};
pub use escape::{escape, escape_os, unescape, unescape_os};
pub use human::{ParseSizeError, parse_human_readable_size};
pub use human::{parse_human_readable_size, ParseSizeError};
pub use pattern::{
InvalidPatternError,
pattern_from_os, pattern_from_bytes,
patterns_from_path, patterns_from_reader, patterns_from_stdin,
pattern_from_bytes, pattern_from_os, patterns_from_path,
patterns_from_reader, patterns_from_stdin, InvalidPatternError,
};
pub use process::{CommandError, CommandReader, CommandReaderBuilder};
pub use wtr::{
StandardStream,
stdout, stdout_buffered_line, stdout_buffered_block,
stdout, stdout_buffered_block, stdout_buffered_line, StandardStream,
};
/// Returns true if and only if stdin is believed to be readable.
@@ -204,8 +203,8 @@ pub use wtr::{
pub fn is_readable_stdin() -> bool {
#[cfg(unix)]
fn imp() -> bool {
use std::os::unix::fs::FileTypeExt;
use same_file::Handle;
use std::os::unix::fs::FileTypeExt;
let ft = match Handle::stdin().and_then(|h| h.as_file().metadata()) {
Err(_) => return false,

65
grep-cli/src/pattern.rs → crates/cli/src/pattern.rs
View File

@@ -2,10 +2,12 @@ use std::error;
use std::ffi::OsStr;
use std::fmt;
use std::fs::File;
use std::io::{self, BufRead};
use std::io;
use std::path::Path;
use std::str;
use bstr::io::BufReadExt;
use escape::{escape, escape_os};
/// An error that occurs when a pattern could not be converted to valid UTF-8.
@@ -27,18 +29,19 @@ impl InvalidPatternError {
}
impl error::Error for InvalidPatternError {
fn description(&self) -> &str { "invalid pattern" }
fn description(&self) -> &str {
"invalid pattern"
}
}
impl fmt::Display for InvalidPatternError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"found invalid UTF-8 in pattern at byte offset {} \
(use hex escape sequences to match arbitrary bytes \
in a pattern, e.g., \\xFF): '{}'",
self.valid_up_to,
self.original,
"found invalid UTF-8 in pattern at byte offset {}: {} \
(disable Unicode mode and use hex escape sequences to match \
arbitrary bytes in a pattern, e.g., '(?-u)\\xFF')",
self.valid_up_to, self.original,
)
}
}
@@ -61,10 +64,7 @@ pub fn pattern_from_os(pattern: &OsStr) -> Result<&str, InvalidPatternError> {
.to_string_lossy()
.find('\u{FFFD}')
.expect("a Unicode replacement codepoint for invalid UTF-8");
InvalidPatternError {
original: escape_os(pattern),
valid_up_to: valid_up_to,
}
InvalidPatternError { original: escape_os(pattern), valid_up_to }
})
}
@@ -77,11 +77,9 @@ pub fn pattern_from_os(pattern: &OsStr) -> Result<&str, InvalidPatternError> {
pub fn pattern_from_bytes(
pattern: &[u8],
) -> Result<&str, InvalidPatternError> {
str::from_utf8(pattern).map_err(|err| {
InvalidPatternError {
original: escape(pattern),
valid_up_to: err.valid_up_to(),
}
str::from_utf8(pattern).map_err(|err| InvalidPatternError {
original: escape(pattern),
valid_up_to: err.valid_up_to(),
})
}
@@ -117,10 +115,7 @@ pub fn patterns_from_stdin() -> io::Result<Vec<String>> {
let stdin = io::stdin();
let locked = stdin.lock();
patterns_from_reader(locked).map_err(|err| {
io::Error::new(
io::ErrorKind::Other,
format!("<stdin>:{}", err),
)
io::Error::new(io::ErrorKind::Other, format!("<stdin>:{}", err))
})
}
@@ -156,28 +151,20 @@ pub fn patterns_from_stdin() -> io::Result<Vec<String>> {
/// ```
pub fn patterns_from_reader<R: io::Read>(rdr: R) -> io::Result<Vec<String>> {
let mut patterns = vec![];
let mut bufrdr = io::BufReader::new(rdr);
let mut line = vec![];
let mut line_number = 0;
while {
line.clear();
io::BufReader::new(rdr).for_byte_line(|line| {
line_number += 1;
bufrdr.read_until(b'\n', &mut line)? > 0
} {
line.pop().unwrap(); // remove trailing '\n'
if line.last() == Some(&b'\r') {
line.pop().unwrap();
}
match pattern_from_bytes(&line) {
Ok(pattern) => patterns.push(pattern.to_string()),
Err(err) => {
return Err(io::Error::new(
io::ErrorKind::Other,
format!("{}: {}", line_number, err),
));
match pattern_from_bytes(line) {
Ok(pattern) => {
patterns.push(pattern.to_string());
Ok(true)
}
Err(err) => Err(io::Error::new(
io::ErrorKind::Other,
format!("{}: {}", line_number, err),
)),
}
}
})?;
Ok(patterns)
}
@@ -195,8 +182,8 @@ mod tests {
#[test]
#[cfg(unix)]
fn os() {
use std::os::unix::ffi::OsStrExt;
use std::ffi::OsStr;
use std::os::unix::ffi::OsStrExt;
let pat = OsStr::from_bytes(b"abc\xFFxyz");
let err = pattern_from_os(pat).unwrap_err();

31
grep-cli/src/process.rs → crates/cli/src/process.rs
View File

@@ -33,7 +33,9 @@ impl CommandError {
}
impl error::Error for CommandError {
fn description(&self) -> &str { "command error" }
fn description(&self) -> &str {
"command error"
}
}
impl fmt::Display for CommandError {
@@ -46,7 +48,12 @@ impl fmt::Display for CommandError {
write!(f, "<stderr is empty>")
} else {
let div = iter::repeat('-').take(79).collect::<String>();
write!(f, "\n{div}\n{msg}\n{div}", div=div, msg=msg.trim())
write!(
f,
"\n{div}\n{msg}\n{div}",
div = div,
msg = msg.trim()
)
}
}
}
@@ -101,12 +108,11 @@ impl CommandReaderBuilder {
.stderr(process::Stdio::piped())
.spawn()?;
let stdout = child.stdout.take().unwrap();
let stderr =
if self.async_stderr {
StderrReader::async(child.stderr.take().unwrap())
} else {
StderrReader::sync(child.stderr.take().unwrap())
};
let stderr = if self.async_stderr {
StderrReader::async(child.stderr.take().unwrap())
} else {
StderrReader::sync(child.stderr.take().unwrap())
};
Ok(CommandReader {
child: child,
stdout: stdout,
@@ -226,9 +232,8 @@ enum StderrReader {
impl StderrReader {
/// Create a reader for stderr that reads contents asynchronously.
fn async(mut stderr: process::ChildStderr) -> StderrReader {
let handle = thread::spawn(move || {
stderr_to_command_error(&mut stderr)
});
let handle =
thread::spawn(move || stderr_to_command_error(&mut stderr));
StderrReader::Async(Some(handle))
}
@@ -247,9 +252,7 @@ impl StderrReader {
let handle = handle
.take()
.expect("read_to_end cannot be called more than once");
handle
.join()
.expect("stderr reading thread does not panic")
handle.join().expect("stderr reading thread does not panic")
}
StderrReader::Sync(ref mut stderr) => {
stderr_to_command_error(stderr)

0
grep-cli/src/wtr.rs → crates/cli/src/wtr.rs
View File

15
crates/core/README.md Normal file
View File

@@ -0,0 +1,15 @@
ripgrep core
------------
This is the core ripgrep crate. In particular, `main.rs` is where the `main`
function lives.
Most of ripgrep core consists of two things:
* The definition of the CLI interface, including docs for every flag.
* Glue code that brings the `grep-matcher`, `grep-regex`, `grep-searcher` and
`grep-printer` crates together to actually execute the search.
Currently, there are no plans to make ripgrep core available as an independent
library. However, much of the heavy lifting of ripgrep is done via its
constituent crates, which can be reused independent of ripgrep. Unfortunately,
there is no guide or tutorial to teach folks how to do this yet.

1644
src/app.rs → crates/core/app.rs
View File

File diff suppressed because it is too large Load Diff

676
src/args.rs → crates/core/args.rs
View File

File diff suppressed because it is too large Load Diff

111
src/config.rs → crates/core/config.rs
View File

@@ -4,12 +4,15 @@
use std::env;
use std::error::Error;
use std::fs::File;
use std::io::{self, BufRead};
use std::ffi::OsString;
use std::fs::File;
use std::io;
use std::path::{Path, PathBuf};
use Result;
use bstr::{io::BufReadExt, ByteSlice};
use log;
use crate::Result;
/// Return a sequence of arguments derived from ripgrep rc configuration files.
pub fn args() -> Vec<OsString> {
@@ -34,7 +37,7 @@ pub fn args() -> Vec<OsString> {
message!("{}:{}", config_path.display(), err);
}
}
debug!(
log::debug!(
"{}: arguments loaded from config file: {:?}",
config_path.display(),
args
@@ -52,7 +55,7 @@ pub fn args() -> Vec<OsString> {
/// for each line in addition to successfully parsed arguments.
fn parse<P: AsRef<Path>>(
path: P,
) -> Result<(Vec<OsString>, Vec<Box<Error>>)> {
) -> Result<(Vec<OsString>, Vec<Box<dyn Error>>)> {
let path = path.as_ref();
match File::open(&path) {
Ok(file) => parse_reader(file),
@@ -73,71 +76,39 @@ fn parse<P: AsRef<Path>>(
/// in addition to successfully parsed arguments.
fn parse_reader<R: io::Read>(
rdr: R,
) -> Result<(Vec<OsString>, Vec<Box<Error>>)> {
let mut bufrdr = io::BufReader::new(rdr);
) -> Result<(Vec<OsString>, Vec<Box<dyn Error>>)> {
let bufrdr = io::BufReader::new(rdr);
let (mut args, mut errs) = (vec![], vec![]);
let mut line = vec![];
let mut line_number = 0;
while {
line.clear();
bufrdr.for_byte_line_with_terminator(|line| {
line_number += 1;
bufrdr.read_until(b'\n', &mut line)? > 0
} {
trim(&mut line);
let line = line.trim();
if line.is_empty() || line[0] == b'#' {
continue;
return Ok(true);
}
match bytes_to_os_string(&line) {
match line.to_os_str() {
Ok(osstr) => {
args.push(osstr);
args.push(osstr.to_os_string());
}
Err(err) => {
errs.push(format!("{}: {}", line_number, err).into());
}
}
}
Ok(true)
})?;
Ok((args, errs))
}
/// Trim the given bytes of whitespace according to the ASCII definition.
fn trim(x: &mut Vec<u8>) {
let upto = x.iter().take_while(|b| is_space(**b)).count();
x.drain(..upto);
let revto = x.len() - x.iter().rev().take_while(|b| is_space(**b)).count();
x.drain(revto..);
}
/// Returns true if and only if the given byte is an ASCII space character.
fn is_space(b: u8) -> bool {
b == b'\t'
|| b == b'\n'
|| b == b'\x0B'
|| b == b'\x0C'
|| b == b'\r'
|| b == b' '
}
/// On Unix, get an OsString from raw bytes.
#[cfg(unix)]
fn bytes_to_os_string(bytes: &[u8]) -> Result<OsString> {
use std::os::unix::ffi::OsStringExt;
Ok(OsString::from_vec(bytes.to_vec()))
}
/// On non-Unix (like Windows), require UTF-8.
#[cfg(not(unix))]
fn bytes_to_os_string(bytes: &[u8]) -> Result<OsString> {
String::from_utf8(bytes.to_vec()).map(OsString::from).map_err(From::from)
}
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use super::parse_reader;
use std::ffi::OsString;
#[test]
fn basic() {
let (args, errs) = parse_reader(&b"\
let (args, errs) = parse_reader(
&b"\
# Test
--context=0
--smart-case
@@ -146,13 +117,13 @@ mod tests {
# --bar
--foo
"[..]).unwrap();
"[..],
)
.unwrap();
assert!(errs.is_empty());
let args: Vec<String> =
args.into_iter().map(|s| s.into_string().unwrap()).collect();
assert_eq!(args, vec![
"--context=0", "--smart-case", "-u", "--foo",
]);
assert_eq!(args, vec!["--context=0", "--smart-case", "-u", "--foo",]);
}
// We test that we can handle invalid UTF-8 on Unix-like systems.
@@ -161,32 +132,38 @@ mod tests {
fn error() {
use std::os::unix::ffi::OsStringExt;
let (args, errs) = parse_reader(&b"\
let (args, errs) = parse_reader(
&b"\
quux
foo\xFFbar
baz
"[..]).unwrap();
"[..],
)
.unwrap();
assert!(errs.is_empty());
assert_eq!(args, vec![
OsString::from("quux"),
OsString::from_vec(b"foo\xFFbar".to_vec()),
OsString::from("baz"),
]);
assert_eq!(
args,
vec![
OsString::from("quux"),
OsString::from_vec(b"foo\xFFbar".to_vec()),
OsString::from("baz"),
]
);
}
// ... but test that invalid UTF-8 fails on Windows.
#[test]
#[cfg(not(unix))]
fn error() {
let (args, errs) = parse_reader(&b"\
let (args, errs) = parse_reader(
&b"\
quux
foo\xFFbar
baz
"[..]).unwrap();
"[..],
)
.unwrap();
assert_eq!(errs.len(), 1);
assert_eq!(args, vec![
OsString::from("quux"),
OsString::from("baz"),
]);
assert_eq!(args, vec![OsString::from("quux"), OsString::from("baz"),]);
}
}

0
src/logger.rs → crates/core/logger.rs
View File

148
src/main.rs → crates/core/main.rs
View File

@@ -1,20 +1,7 @@
#[macro_use]
extern crate clap;
extern crate grep;
extern crate ignore;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate log;
extern crate num_cpus;
extern crate regex;
#[macro_use]
extern crate serde_json;
extern crate termcolor;
use std::error;
use std::io::{self, Write};
use std::process;
use std::sync::{Arc, Mutex};
use std::sync::Mutex;
use std::time::Instant;
use ignore::WalkState;
@@ -33,36 +20,63 @@ mod path_printer;
mod search;
mod subject;
type Result<T> = ::std::result::Result<T, Box<::std::error::Error>>;
// Since Rust no longer uses jemalloc by default, ripgrep will, by default,
// use the system allocator. On Linux, this would normally be glibc's
// allocator, which is pretty good. In particular, ripgrep does not have a
// particularly allocation heavy workload, so there really isn't much
// difference (for ripgrep's purposes) between glibc's allocator and jemalloc.
//
// However, when ripgrep is built with musl, this means ripgrep will use musl's
// allocator, which appears to be substantially worse. (musl's goal is not to
// have the fastest version of everything. Its goal is to be small and amenable
// to static compilation.) Even though ripgrep isn't particularly allocation
// heavy, musl's allocator appears to slow down ripgrep quite a bit. Therefore,
// when building with musl, we use jemalloc.
//
// We don't unconditionally use jemalloc because it can be nice to use the
// system's default allocator by default. Moreover, jemalloc seems to increase
// compilation times by a bit.
//
// Moreover, we only do this on 64-bit systems since jemalloc doesn't support
// i686.
#[cfg(all(target_env = "musl", target_pointer_width = "64"))]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
type Result<T> = ::std::result::Result<T, Box<dyn error::Error>>;
fn main() {
match Args::parse().and_then(try_main) {
Ok(true) => process::exit(0),
Ok(false) => process::exit(1),
Err(err) => {
eprintln!("{}", err);
process::exit(2);
}
if let Err(err) = Args::parse().and_then(try_main) {
eprintln!("{}", err);
process::exit(2);
}
}
fn try_main(args: Args) -> Result<bool> {
fn try_main(args: Args) -> Result<()> {
use args::Command::*;
match args.command()? {
Search => search(args),
SearchParallel => search_parallel(args),
let matched = match args.command()? {
Search => search(&args),
SearchParallel => search_parallel(&args),
SearchNever => Ok(false),
Files => files(args),
FilesParallel => files_parallel(args),
Types => types(args),
Files => files(&args),
FilesParallel => files_parallel(&args),
Types => types(&args),
PCRE2Version => pcre2_version(&args),
}?;
if matched && (args.quiet() || !messages::errored()) {
process::exit(0)
} else if messages::errored() {
process::exit(2)
} else {
process::exit(1)
}
}
/// The top-level entry point for single-threaded search. This recursively
/// steps through the file list (current directory by default) and searches
/// each file sequentially.
fn search(args: Args) -> Result<bool> {
fn search(args: &Args) -> Result<bool> {
let started_at = Instant::now();
let quit_after_match = args.quit_after_match()?;
let subject_builder = args.subject_builder();
@@ -82,7 +96,7 @@ fn search(args: Args) -> Result<bool> {
if err.kind() == io::ErrorKind::BrokenPipe {
break;
}
message!("{}: {}", subject.path().display(), err);
err_message!("{}: {}", subject.path().display(), err);
continue;
}
};
@@ -105,30 +119,27 @@ fn search(args: Args) -> Result<bool> {
/// The top-level entry point for multi-threaded search. The parallelism is
/// itself achieved by the recursive directory traversal. All we need to do is
/// feed it a worker for performing a search on each file.
fn search_parallel(args: Args) -> Result<bool> {
fn search_parallel(args: &Args) -> Result<bool> {
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering::SeqCst;
let quit_after_match = args.quit_after_match()?;
let started_at = Instant::now();
let subject_builder = Arc::new(args.subject_builder());
let bufwtr = Arc::new(args.buffer_writer()?);
let stats = Arc::new(args.stats()?.map(Mutex::new));
let matched = Arc::new(AtomicBool::new(false));
let subject_builder = args.subject_builder();
let bufwtr = args.buffer_writer()?;
let stats = args.stats()?.map(Mutex::new);
let matched = AtomicBool::new(false);
let mut searcher_err = None;
args.walker_parallel()?.run(|| {
let args = args.clone();
let bufwtr = Arc::clone(&bufwtr);
let stats = Arc::clone(&stats);
let matched = Arc::clone(&matched);
let subject_builder = Arc::clone(&subject_builder);
let bufwtr = &bufwtr;
let stats = &stats;
let matched = &matched;
let subject_builder = &subject_builder;
let mut searcher = match args.search_worker(bufwtr.buffer()) {
Ok(searcher) => searcher,
Err(err) => {
searcher_err = Some(err);
return Box::new(move |_| {
WalkState::Quit
});
return Box::new(move |_| WalkState::Quit);
}
};
@@ -141,7 +152,7 @@ fn search_parallel(args: Args) -> Result<bool> {
let search_result = match searcher.search(&subject) {
Ok(search_result) => search_result,
Err(err) => {
message!("{}: {}", subject.path().display(), err);
err_message!("{}: {}", subject.path().display(), err);
return WalkState::Continue;
}
};
@@ -158,7 +169,7 @@ fn search_parallel(args: Args) -> Result<bool> {
return WalkState::Quit;
}
// Otherwise, we continue on our merry way.
message!("{}: {}", subject.path().display(), err);
err_message!("{}: {}", subject.path().display(), err);
}
if matched.load(SeqCst) && quit_after_match {
WalkState::Quit
@@ -170,7 +181,7 @@ fn search_parallel(args: Args) -> Result<bool> {
if let Some(err) = searcher_err.take() {
return Err(err);
}
if let Some(ref locked_stats) = *stats {
if let Some(ref locked_stats) = stats {
let elapsed = Instant::now().duration_since(started_at);
let stats = locked_stats.lock().unwrap();
let mut searcher = args.search_worker(args.stdout())?;
@@ -183,7 +194,7 @@ fn search_parallel(args: Args) -> Result<bool> {
/// The top-level entry point for listing files without searching them. This
/// recursively steps through the file list (current directory by default) and
/// prints each path sequentially using a single thread.
fn files(args: Args) -> Result<bool> {
fn files(args: &Args) -> Result<bool> {
let quit_after_match = args.quit_after_match()?;
let subject_builder = args.subject_builder();
let mut matched = false;
@@ -213,16 +224,16 @@ fn files(args: Args) -> Result<bool> {
/// The top-level entry point for listing files without searching them. This
/// recursively steps through the file list (current directory by default) and
/// prints each path sequentially using multiple threads.
fn files_parallel(args: Args) -> Result<bool> {
fn files_parallel(args: &Args) -> Result<bool> {
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering::SeqCst;
use std::sync::mpsc;
use std::thread;
let quit_after_match = args.quit_after_match()?;
let subject_builder = Arc::new(args.subject_builder());
let subject_builder = args.subject_builder();
let mut path_printer = args.path_printer(args.stdout())?;
let matched = Arc::new(AtomicBool::new(false));
let matched = AtomicBool::new(false);
let (tx, rx) = mpsc::channel::<Subject>();
let print_thread = thread::spawn(move || -> io::Result<()> {
@@ -232,8 +243,8 @@ fn files_parallel(args: Args) -> Result<bool> {
Ok(())
});
args.walker_parallel()?.run(|| {
let subject_builder = Arc::clone(&subject_builder);
let matched = Arc::clone(&matched);
let subject_builder = &subject_builder;
let matched = &matched;
let tx = tx.clone();
Box::new(move |result| {
@@ -265,7 +276,7 @@ fn files_parallel(args: Args) -> Result<bool> {
}
/// The top-level entry point for --type-list.
fn types(args: Args) -> Result<bool> {
fn types(args: &Args) -> Result<bool> {
let mut count = 0;
let mut stdout = args.stdout();
for def in args.type_defs()? {
@@ -285,3 +296,30 @@ fn types(args: Args) -> Result<bool> {
}
Ok(count > 0)
}
/// The top-level entry point for --pcre2-version.
fn pcre2_version(args: &Args) -> Result<bool> {
#[cfg(feature = "pcre2")]
fn imp(args: &Args) -> Result<bool> {
use grep::pcre2;
let mut stdout = args.stdout();
let (major, minor) = pcre2::version();
writeln!(stdout, "PCRE2 {}.{} is available", major, minor)?;
if cfg!(target_pointer_width = "64") && pcre2::is_jit_available() {
writeln!(stdout, "JIT is available")?;
}
Ok(true)
}
#[cfg(not(feature = "pcre2"))]
fn imp(args: &Args) -> Result<bool> {
let mut stdout = args.stdout();
writeln!(stdout, "PCRE2 is not available in this build of ripgrep.")?;
Ok(false)
}
imp(args)
}

34
src/messages.rs → crates/core/messages.rs
View File

@@ -1,21 +1,35 @@
use std::sync::atomic::{ATOMIC_BOOL_INIT, AtomicBool, Ordering};
use std::sync::atomic::{AtomicBool, Ordering};
static MESSAGES: AtomicBool = ATOMIC_BOOL_INIT;
static IGNORE_MESSAGES: AtomicBool = ATOMIC_BOOL_INIT;
static MESSAGES: AtomicBool = AtomicBool::new(false);
static IGNORE_MESSAGES: AtomicBool = AtomicBool::new(false);
static ERRORED: AtomicBool = AtomicBool::new(false);
/// Emit a non-fatal error message, unless messages were disabled.
#[macro_export]
macro_rules! message {
($($tt:tt)*) => {
if ::messages::messages() {
if crate::messages::messages() {
eprintln!($($tt)*);
}
}
}
/// Like message, but sets ripgrep's "errored" flag, which controls the exit
/// status.
#[macro_export]
macro_rules! err_message {
($($tt:tt)*) => {
crate::messages::set_errored();
message!($($tt)*);
}
}
/// Emit a non-fatal ignore-related error message (like a parse error), unless
/// ignore-messages were disabled.
#[macro_export]
macro_rules! ignore_message {
($($tt:tt)*) => {
if ::messages::messages() && ::messages::ignore_messages() {
if crate::messages::messages() && crate::messages::ignore_messages() {
eprintln!($($tt)*);
}
}
@@ -48,3 +62,13 @@ pub fn ignore_messages() -> bool {
pub fn set_ignore_messages(yes: bool) {
IGNORE_MESSAGES.store(yes, Ordering::SeqCst)
}
/// Returns true if and only if ripgrep came across a non-fatal error.
pub fn errored() -> bool {
ERRORED.load(Ordering::SeqCst)
}
/// Indicate that ripgrep has come across a non-fatal error.
pub fn set_errored() {
ERRORED.store(true, Ordering::SeqCst);
}

5
src/path_printer.rs → crates/core/path_printer.rs
View File

@@ -37,10 +37,7 @@ impl PathPrinterBuilder {
/// Create a new path printer with the current configuration that writes
/// paths to the given writer.
pub fn build<W: WriteColor>(&self, wtr: W) -> PathPrinter<W> {
PathPrinter {
config: self.config.clone(),
wtr: wtr,
}
PathPrinter { config: self.config.clone(), wtr }
}
/// Set the color specification for this printer.

138
src/search.rs → crates/core/search.rs
View File

@@ -7,15 +7,16 @@ use std::time::Duration;
use grep::cli;
use grep::matcher::Matcher;
#[cfg(feature = "pcre2")]
use grep::pcre2::{RegexMatcher as PCRE2RegexMatcher};
use grep::printer::{JSON, Standard, Summary, Stats};
use grep::regex::{RegexMatcher as RustRegexMatcher};
use grep::searcher::Searcher;
use grep::pcre2::RegexMatcher as PCRE2RegexMatcher;
use grep::printer::{Standard, Stats, Summary, JSON};
use grep::regex::RegexMatcher as RustRegexMatcher;
use grep::searcher::{BinaryDetection, Searcher};
use ignore::overrides::Override;
use serde_json as json;
use serde_json::json;
use termcolor::WriteColor;
use subject::Subject;
use crate::subject::Subject;
/// The configuration for the search worker. Among a few other things, the
/// configuration primarily controls the way we show search results to users
@@ -26,6 +27,8 @@ struct Config {
preprocessor: Option<PathBuf>,
preprocessor_globs: Override,
search_zip: bool,
binary_implicit: BinaryDetection,
binary_explicit: BinaryDetection,
}
impl Default for Config {
@@ -35,6 +38,8 @@ impl Default for Config {
preprocessor: None,
preprocessor_globs: Override::empty(),
search_zip: false,
binary_implicit: BinaryDetection::none(),
binary_explicit: BinaryDetection::none(),
}
}
}
@@ -65,7 +70,7 @@ impl SearchWorkerBuilder {
SearchWorkerBuilder {
config: Config::default(),
command_builder: cmd_builder,
decomp_builder: decomp_builder,
decomp_builder,
}
}
@@ -81,8 +86,12 @@ impl SearchWorkerBuilder {
let command_builder = self.command_builder.clone();
let decomp_builder = self.decomp_builder.clone();
SearchWorker {
config, command_builder, decomp_builder,
matcher, searcher, printer,
config,
command_builder,
decomp_builder,
matcher,
searcher,
printer,
}
}
@@ -133,6 +142,37 @@ impl SearchWorkerBuilder {
self.config.search_zip = yes;
self
}
/// Set the binary detection that should be used when searching files
/// found via a recursive directory search.
///
/// Generally, this binary detection may be `BinaryDetection::quit` if
/// we want to skip binary files completely.
///
/// By default, no binary detection is performed.
pub fn binary_detection_implicit(
&mut self,
detection: BinaryDetection,
) -> &mut SearchWorkerBuilder {
self.config.binary_implicit = detection;
self
}
/// Set the binary detection that should be used when searching files
/// explicitly supplied by an end user.
///
/// Generally, this binary detection should NOT be `BinaryDetection::quit`,
/// since we never want to automatically filter files supplied by the end
/// user.
///
/// By default, no binary detection is performed.
pub fn binary_detection_explicit(
&mut self,
detection: BinaryDetection,
) -> &mut SearchWorkerBuilder {
self.config.binary_explicit = detection;
self
}
}
/// The result of executing a search.
@@ -191,9 +231,7 @@ impl<W: WriteColor> Printer<W> {
stats: &Stats,
) -> io::Result<()> {
match *self {
Printer::JSON(_) => {
self.print_stats_json(total_duration, stats)
}
Printer::JSON(_) => self.print_stats_json(total_duration, stats),
Printer::Standard(_) | Printer::Summary(_) => {
self.print_stats_human(total_duration, stats)
}
@@ -237,17 +275,20 @@ impl<W: WriteColor> Printer<W> {
// the grep-printer crate. We simply "extend" it with the 'summary'
// message type.
let fractional = fractional_seconds(total_duration);
json::to_writer(self.get_mut(), &json!({
"type": "summary",
"data": {
"stats": stats,
"elapsed_total": {
"secs": total_duration.as_secs(),
"nanos": total_duration.subsec_nanos(),
"human": format!("{:0.6}s", fractional),
},
}
}))?;
json::to_writer(
self.get_mut(),
&json!({
"type": "summary",
"data": {
"stats": stats,
"elapsed_total": {
"secs": total_duration.as_secs(),
"nanos": total_duration.subsec_nanos(),
"human": format!("{:0.6}s", fractional),
},
}
}),
)?;
write!(self.get_mut(), "\n")
}
@@ -279,7 +320,23 @@ pub struct SearchWorker<W> {
impl<W: WriteColor> SearchWorker<W> {
/// Execute a search over the given subject.
pub fn search(&mut self, subject: &Subject) -> io::Result<SearchResult> {
self.search_impl(subject)
let bin = if subject.is_explicit() {
self.config.binary_explicit.clone()
} else {
self.config.binary_implicit.clone()
};
self.searcher.set_binary_detection(bin);
let path = subject.path();
if subject.is_stdin() {
self.search_reader(path, io::stdin().lock())
} else if self.should_preprocess(path) {
self.search_preprocessor(path)
} else if self.should_decompress(path) {
self.search_decompress(path)
} else {
self.search_path(path)
}
}
/// Return a mutable reference to the underlying printer.
@@ -305,22 +362,6 @@ impl<W: WriteColor> SearchWorker<W> {
}
}
/// Search the given subject using the appropriate strategy.
fn search_impl(&mut self, subject: &Subject) -> io::Result<SearchResult> {
let path = subject.path();
if subject.is_stdin() {
let stdin = io::stdin();
// A `return` here appeases the borrow checker. NLL will fix this.
return self.search_reader(path, stdin.lock());
} else if self.should_preprocess(path) {
self.search_preprocessor(path)
} else if self.should_decompress(path) {
self.search_decompress(path)
} else {
self.search_path(path)
}
}
/// Returns true if and only if the given file path should be
/// decompressed before searching.
fn should_decompress(&self, path: &Path) -> bool {
@@ -348,11 +389,19 @@ impl<W: WriteColor> SearchWorker<W> {
&mut self,
path: &Path,
) -> io::Result<SearchResult> {
let bin = self.config.preprocessor.clone().unwrap();
let mut cmd = Command::new(&bin);
let bin = self.config.preprocessor.as_ref().unwrap();
let mut cmd = Command::new(bin);
cmd.arg(path).stdin(Stdio::from(File::open(path)?));
let rdr = self.command_builder.build(&mut cmd)?;
let rdr = self.command_builder.build(&mut cmd).map_err(|err| {
io::Error::new(
io::ErrorKind::Other,
format!(
"preprocessor command could not start: '{:?}': {}",
cmd, err,
),
)
})?;
self.search_reader(path, rdr).map_err(|err| {
io::Error::new(
io::ErrorKind::Other,
@@ -364,10 +413,7 @@ impl<W: WriteColor> SearchWorker<W> {
/// Attempt to decompress the data at the given file path and search the
/// result. If the given file path isn't recognized as a compressed file,
/// then search it without doing any decompression.
fn search_decompress(
&mut self,
path: &Path,
) -> io::Result<SearchResult> {
fn search_decompress(&mut self, path: &Path) -> io::Result<SearchResult> {
let rdr = self.decomp_builder.build(path)?;
self.search_reader(path, rdr)
}

68
src/subject.rs → crates/core/subject.rs
View File

@@ -1,6 +1,7 @@
use std::path::Path;
use ignore::{self, DirEntry};
use log;
/// A configuration for describing how subjects should be built.
#[derive(Clone, Debug)]
@@ -10,9 +11,7 @@ struct Config {
impl Default for Config {
fn default() -> Config {
Config {
strip_dot_prefix: false,
}
Config { strip_dot_prefix: false }
}
}
@@ -40,7 +39,7 @@ impl SubjectBuilder {
match result {
Ok(dent) => self.build(dent),
Err(err) => {
message!("{}", err);
err_message!("{}", err);
None
}
}
@@ -51,24 +50,17 @@ impl SubjectBuilder {
/// If a subject could not be created or should otherwise not be searched,
/// then this returns `None` after emitting any relevant log messages.
pub fn build(&self, dent: DirEntry) -> Option<Subject> {
let subj = Subject {
dent: dent,
strip_dot_prefix: self.config.strip_dot_prefix,
};
let subj =
Subject { dent, strip_dot_prefix: self.config.strip_dot_prefix };
if let Some(ignore_err) = subj.dent.error() {
ignore_message!("{}", ignore_err);
}
// If this entry represents stdin, then we always search it.
if subj.dent.is_stdin() {
// If this entry was explicitly provided by an end user, then we always
// want to search it.
if subj.is_explicit() {
return Some(subj);
}
// If this subject has a depth of 0, then it was provided explicitly
// by an end user (or via a shell glob). In this case, we always want
// to search it if it even smells like a file (e.g., a symlink).
if subj.dent.depth() == 0 && !subj.is_dir() {
return Some(subj);
}
// At this point, we only want to search something it's explicitly a
// At this point, we only want to search something if it's explicitly a
// file. This omits symlinks. (If ripgrep was configured to follow
// symlinks, then they have already been followed by the directory
// traversal.)
@@ -79,12 +71,12 @@ impl SubjectBuilder {
// directory. Otherwise, emitting messages for directories is just
// noisy.
if !subj.is_dir() {
debug!(
log::debug!(
"ignoring {}: failed to pass subject filter: \
file type: {:?}, metadata: {:?}",
subj.dent.path().display(),
subj.dent.file_type(),
subj.dent.metadata()
subj.dent.path().display(),
subj.dent.file_type(),
subj.dent.metadata()
);
}
None
@@ -126,9 +118,39 @@ impl Subject {
self.dent.is_stdin()
}
/// Returns true if and only if this subject points to a directory.
/// Returns true if and only if this entry corresponds to a subject to
/// search that was explicitly supplied by an end user.
///
/// Generally, this corresponds to either stdin or an explicit file path
/// argument. e.g., in `rg foo some-file ./some-dir/`, `some-file` is
/// an explicit subject, but, e.g., `./some-dir/some-other-file` is not.
///
/// However, note that ripgrep does not see through shell globbing. e.g.,
/// in `rg foo ./some-dir/*`, `./some-dir/some-other-file` will be treated
/// as an explicit subject.
pub fn is_explicit(&self) -> bool {
// stdin is obvious. When an entry has a depth of 0, that means it
// was explicitly provided to our directory iterator, which means it
// was in turn explicitly provided by the end user. The !is_dir check
// means that we want to search files even if their symlinks, again,
// because they were explicitly provided. (And we never want to try
// to search a directory.)
self.is_stdin() || (self.dent.depth() == 0 && !self.is_dir())
}
/// Returns true if and only if this subject points to a directory after
/// following symbolic links.
fn is_dir(&self) -> bool {
self.dent.file_type().map_or(false, |ft| ft.is_dir())
let ft = match self.dent.file_type() {
None => return false,
Some(ft) => ft,
};
if ft.is_dir() {
return true;
}
// If this is a symlink, then we want to follow it to determine
// whether it's a directory or not.
self.dent.path_is_symlink() && self.dent.path().is_dir()
}
/// Returns true if and only if this subject points to a file.

0
globset/COPYING → crates/globset/COPYING
View File

18
globset/Cargo.toml → crates/globset/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "globset"
version = "0.4.2" #:version
version = "0.4.5" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """
Cross platform single glob and glob set matching. Glob set matching is the
@@ -8,8 +8,8 @@ process of matching one or more glob patterns against a single candidate path
simultaneously, and returning all of the globs that matched.
"""
documentation = "https://docs.rs/globset"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/globset"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/globset"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/globset"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/globset"
readme = "README.md"
keywords = ["regex", "glob", "multiple", "set", "pattern"]
license = "Unlicense/MIT"
@@ -19,14 +19,18 @@ name = "globset"
bench = false
[dependencies]
aho-corasick = "0.6.8"
aho-corasick = "0.7.3"
bstr = { version = "0.2.0", default-features = false, features = ["std"] }
fnv = "1.0.6"
log = "0.4.5"
memchr = "2.0.2"
regex = "1.0.5"
regex = "1.1.5"
serde = { version = "1.0.104", optional = true }
[dev-dependencies]
glob = "0.2.11"
glob = "0.3.0"
lazy_static = "1"
serde_json = "1.0.45"
[features]
simd-accel = []
serde1 = ["serde"]

0
grep-cli/LICENSE-MIT → crates/globset/LICENSE-MIT
View File

4
globset/README.md → crates/globset/README.md
View File

@@ -29,6 +29,10 @@ and this to your crate root:
extern crate globset;
```
### Features
* `serde1`: Enables implementing Serde traits on the `Glob` type.
### Example: one glob
This example shows how to match a single glob against a single file path.

0
grep-cli/UNLICENSE → crates/globset/UNLICENSE
View File

5
globset/benches/bench.rs → crates/globset/benches/bench.rs
View File

@@ -6,14 +6,9 @@ tool itself, see the benchsuite directory.
extern crate glob;
extern crate globset;
#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate test;
use std::ffi::OsStr;
use std::path::Path;
use globset::{Candidate, Glob, GlobMatcher, GlobSet, GlobSetBuilder};
const EXT: &'static str = "some/a/bigger/path/to/the/crazy/needle.txt";

301
globset/src/glob.rs → crates/globset/src/glob.rs
View File

@@ -2,13 +2,13 @@ use std::fmt;
use std::hash;
use std::iter;
use std::ops::{Deref, DerefMut};
use std::path::{Path, is_separator};
use std::path::{is_separator, Path};
use std::str;
use regex;
use regex::bytes::Regex;
use {Candidate, Error, ErrorKind, new_regex};
use {new_regex, Candidate, Error, ErrorKind};
/// Describes a matching strategy for a particular pattern.
///
@@ -85,16 +85,16 @@ pub struct Glob {
}
impl PartialEq for Glob {
fn eq(&self, other: &Glob) -> bool {
self.glob == other.glob && self.opts == other.opts
}
fn eq(&self, other: &Glob) -> bool {
self.glob == other.glob && self.opts == other.opts
}
}
impl hash::Hash for Glob {
fn hash<H: hash::Hasher>(&self, state: &mut H) {
self.glob.hash(state);
self.opts.hash(state);
}
fn hash<H: hash::Hasher>(&self, state: &mut H) {
self.glob.hash(state);
self.opts.hash(state);
}
}
impl fmt::Display for Glob {
@@ -103,6 +103,14 @@ impl fmt::Display for Glob {
}
}
impl str::FromStr for Glob {
type Err = Error;
fn from_str(glob: &str) -> Result<Self, Self::Err> {
Self::new(glob)
}
}
/// A matcher for a single pattern.
#[derive(Clone, Debug)]
pub struct GlobMatcher {
@@ -122,6 +130,11 @@ impl GlobMatcher {
pub fn is_match_candidate(&self, path: &Candidate) -> bool {
self.re.is_match(&path.path)
}
/// Returns the `Glob` used to compile this matcher.
pub fn glob(&self) -> &Glob {
&self.pat
}
}
/// A strategic matcher for a single pattern.
@@ -214,11 +227,15 @@ struct Tokens(Vec<Token>);
impl Deref for Tokens {
type Target = Vec<Token>;
fn deref(&self) -> &Vec<Token> { &self.0 }
fn deref(&self) -> &Vec<Token> {
&self.0
}
}
impl DerefMut for Tokens {
fn deref_mut(&mut self) -> &mut Vec<Token> { &mut self.0 }
fn deref_mut(&mut self) -> &mut Vec<Token> {
&mut self.0
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
@@ -229,10 +246,7 @@ enum Token {
RecursivePrefix,
RecursiveSuffix,
RecursiveZeroOrMore,
Class {
negated: bool,
ranges: Vec<(char, char)>,
},
Class { negated: bool, ranges: Vec<(char, char)> },
Alternates(Vec<Tokens>),
}
@@ -244,12 +258,9 @@ impl Glob {
/// Returns a matcher for this pattern.
pub fn compile_matcher(&self) -> GlobMatcher {
let re = new_regex(&self.re)
.expect("regex compilation shouldn't fail");
GlobMatcher {
pat: self.clone(),
re: re,
}
let re =
new_regex(&self.re).expect("regex compilation shouldn't fail");
GlobMatcher { pat: self.clone(), re: re }
}
/// Returns a strategic matcher.
@@ -260,13 +271,9 @@ impl Glob {
#[cfg(test)]
fn compile_strategic_matcher(&self) -> GlobStrategic {
let strategy = MatchStrategy::new(self);
let re = new_regex(&self.re)
.expect("regex compilation shouldn't fail");
GlobStrategic {
strategy: strategy,
pat: self.clone(),
re: re,
}
let re =
new_regex(&self.re).expect("regex compilation shouldn't fail");
GlobStrategic { strategy: strategy, pat: self.clone(), re: re }
}
/// Returns the original glob pattern used to build this pattern.
@@ -524,7 +531,7 @@ impl Glob {
| Token::RecursiveZeroOrMore => {
return None;
}
Token::Class{..} | Token::Alternates(..) => {
Token::Class { .. } | Token::Alternates(..) => {
// We *could* be a little smarter here, but either one
// of these is going to prevent our literal optimizations
// anyway, so give up.
@@ -561,10 +568,7 @@ impl<'a> GlobBuilder<'a> {
///
/// The pattern is not compiled until `build` is called.
pub fn new(glob: &'a str) -> GlobBuilder<'a> {
GlobBuilder {
glob: glob,
opts: GlobOptions::default(),
}
GlobBuilder { glob: glob, opts: GlobOptions::default() }
}
/// Parses and builds the pattern.
@@ -837,40 +841,63 @@ impl<'a> Parser<'a> {
fn parse_star(&mut self) -> Result<(), Error> {
let prev = self.prev;
if self.chars.peek() != Some(&'*') {
if self.peek() != Some('*') {
self.push_token(Token::ZeroOrMore)?;
return Ok(());
}
assert!(self.bump() == Some('*'));
if !self.have_tokens()? {
self.push_token(Token::RecursivePrefix)?;
let next = self.bump();
if !next.map(is_separator).unwrap_or(true) {
return Err(self.error(ErrorKind::InvalidRecursive));
if !self.peek().map_or(true, is_separator) {
self.push_token(Token::ZeroOrMore)?;
self.push_token(Token::ZeroOrMore)?;
} else {
self.push_token(Token::RecursivePrefix)?;
assert!(self.bump().map_or(true, is_separator));
}
return Ok(());
}
self.pop_token()?;
if !prev.map(is_separator).unwrap_or(false) {
if self.stack.len() <= 1
|| (prev != Some(',') && prev != Some('{')) {
return Err(self.error(ErrorKind::InvalidRecursive));
|| (prev != Some(',') && prev != Some('{'))
{
self.push_token(Token::ZeroOrMore)?;
self.push_token(Token::ZeroOrMore)?;
return Ok(());
}
}
match self.chars.peek() {
let is_suffix = match self.peek() {
None => {
assert!(self.bump().is_none());
self.push_token(Token::RecursiveSuffix)
true
}
Some(&',') | Some(&'}') if self.stack.len() >= 2 => {
self.push_token(Token::RecursiveSuffix)
}
Some(&c) if is_separator(c) => {
Some(',') | Some('}') if self.stack.len() >= 2 => true,
Some(c) if is_separator(c) => {
assert!(self.bump().map(is_separator).unwrap_or(false));
self.push_token(Token::RecursiveZeroOrMore)
false
}
_ => {
self.push_token(Token::ZeroOrMore)?;
self.push_token(Token::ZeroOrMore)?;
return Ok(());
}
};
match self.pop_token()? {
Token::RecursivePrefix => {
self.push_token(Token::RecursivePrefix)?;
}
Token::RecursiveSuffix => {
self.push_token(Token::RecursiveSuffix)?;
}
_ => {
if is_suffix {
self.push_token(Token::RecursiveSuffix)?;
} else {
self.push_token(Token::RecursiveZeroOrMore)?;
}
}
_ => Err(self.error(ErrorKind::InvalidRecursive)),
}
Ok(())
}
fn parse_class(&mut self) -> Result<(), Error> {
@@ -934,7 +961,10 @@ impl<'a> Parser<'a> {
// invariant: in_range is only set when there is
// already at least one character seen.
add_to_last_range(
&self.glob, ranges.last_mut().unwrap(), c)?;
&self.glob,
ranges.last_mut().unwrap(),
c,
)?;
} else {
ranges.push((c, c));
}
@@ -948,10 +978,7 @@ impl<'a> Parser<'a> {
// it as a literal.
ranges.push(('-', '-'));
}
self.push_token(Token::Class {
negated: negated,
ranges: ranges,
})
self.push_token(Token::Class { negated: negated, ranges: ranges })
}
fn bump(&mut self) -> Option<char> {
@@ -959,6 +986,10 @@ impl<'a> Parser<'a> {
self.cur = self.chars.next();
self.cur
}
fn peek(&mut self) -> Option<char> {
self.chars.peek().map(|&ch| ch)
}
}
#[cfg(test)]
@@ -976,9 +1007,9 @@ fn ends_with(needle: &[u8], haystack: &[u8]) -> bool {
#[cfg(test)]
mod tests {
use {GlobSetBuilder, ErrorKind};
use super::{Glob, GlobBuilder, Token};
use super::Token::*;
use super::{Glob, GlobBuilder, Token};
use {ErrorKind, GlobSetBuilder};
#[derive(Clone, Copy, Debug, Default)]
struct Options {
@@ -994,7 +1025,7 @@ mod tests {
let pat = Glob::new($pat).unwrap();
assert_eq!($tokens, pat.tokens.0);
}
}
};
}
macro_rules! syntaxerr {
@@ -1004,7 +1035,7 @@ mod tests {
let err = Glob::new($pat).unwrap_err();
assert_eq!(&$err, err.kind());
}
}
};
}
macro_rules! toregex {
@@ -1086,7 +1117,9 @@ mod tests {
};
}
fn s(string: &str) -> String { string.to_string() }
fn s(string: &str) -> String {
string.to_string()
}
fn class(s: char, e: char) -> Token {
Class { negated: false, ranges: vec![(s, e)] }
@@ -1110,16 +1143,20 @@ mod tests {
syntax!(any2, "a?b", vec![Literal('a'), Any, Literal('b')]);
syntax!(seq1, "*", vec![ZeroOrMore]);
syntax!(seq2, "a*b", vec![Literal('a'), ZeroOrMore, Literal('b')]);
syntax!(seq3, "*a*b*", vec![
ZeroOrMore, Literal('a'), ZeroOrMore, Literal('b'), ZeroOrMore,
]);
syntax!(
seq3,
"*a*b*",
vec![ZeroOrMore, Literal('a'), ZeroOrMore, Literal('b'), ZeroOrMore,]
);
syntax!(rseq1, "**", vec![RecursivePrefix]);
syntax!(rseq2, "**/", vec![RecursivePrefix]);
syntax!(rseq3, "/**", vec![RecursiveSuffix]);
syntax!(rseq4, "/**/", vec![RecursiveZeroOrMore]);
syntax!(rseq5, "a/**/b", vec![
Literal('a'), RecursiveZeroOrMore, Literal('b'),
]);
syntax!(
rseq5,
"a/**/b",
vec![Literal('a'), RecursiveZeroOrMore, Literal('b'),]
);
syntax!(cls1, "[a]", vec![class('a', 'a')]);
syntax!(cls2, "[!a]", vec![classn('a', 'a')]);
syntax!(cls3, "[a-z]", vec![class('a', 'z')]);
@@ -1131,9 +1168,11 @@ mod tests {
syntax!(cls9, "[a-]", vec![rclass(&[('a', 'a'), ('-', '-')])]);
syntax!(cls10, "[-a-z]", vec![rclass(&[('-', '-'), ('a', 'z')])]);
syntax!(cls11, "[a-z-]", vec![rclass(&[('a', 'z'), ('-', '-')])]);
syntax!(cls12, "[-a-z-]", vec![
rclass(&[('-', '-'), ('a', 'z'), ('-', '-')]),
]);
syntax!(
cls12,
"[-a-z-]",
vec![rclass(&[('-', '-'), ('a', 'z'), ('-', '-')]),]
);
syntax!(cls13, "[]-z]", vec![class(']', 'z')]);
syntax!(cls14, "[--z]", vec![class('-', 'z')]);
syntax!(cls15, "[ --]", vec![class(' ', '-')]);
@@ -1144,13 +1183,6 @@ mod tests {
syntax!(cls20, "[^a]", vec![classn('a', 'a')]);
syntax!(cls21, "[^a-z]", vec![classn('a', 'z')]);
syntaxerr!(err_rseq1, "a**", ErrorKind::InvalidRecursive);
syntaxerr!(err_rseq2, "**a", ErrorKind::InvalidRecursive);
syntaxerr!(err_rseq3, "a**b", ErrorKind::InvalidRecursive);
syntaxerr!(err_rseq4, "***", ErrorKind::InvalidRecursive);
syntaxerr!(err_rseq5, "/a**", ErrorKind::InvalidRecursive);
syntaxerr!(err_rseq6, "/**a", ErrorKind::InvalidRecursive);
syntaxerr!(err_rseq7, "/a**b", ErrorKind::InvalidRecursive);
syntaxerr!(err_unclosed1, "[", ErrorKind::UnclosedClass);
syntaxerr!(err_unclosed2, "[]", ErrorKind::UnclosedClass);
syntaxerr!(err_unclosed3, "[!", ErrorKind::UnclosedClass);
@@ -1158,26 +1190,14 @@ mod tests {
syntaxerr!(err_range1, "[z-a]", ErrorKind::InvalidRange('z', 'a'));
syntaxerr!(err_range2, "[z--]", ErrorKind::InvalidRange('z', '-'));
const CASEI: Options = Options {
casei: Some(true),
litsep: None,
bsesc: None,
};
const SLASHLIT: Options = Options {
casei: None,
litsep: Some(true),
bsesc: None,
};
const NOBSESC: Options = Options {
casei: None,
litsep: None,
bsesc: Some(false),
};
const BSESC: Options = Options {
casei: None,
litsep: None,
bsesc: Some(true),
};
const CASEI: Options =
Options { casei: Some(true), litsep: None, bsesc: None };
const SLASHLIT: Options =
Options { casei: None, litsep: Some(true), bsesc: None };
const NOBSESC: Options =
Options { casei: None, litsep: None, bsesc: Some(false) };
const BSESC: Options =
Options { casei: None, litsep: None, bsesc: Some(true) };
toregex!(re_casei, "a", "(?i)^a$", &CASEI);
@@ -1194,8 +1214,30 @@ mod tests {
toregex!(re8, "[*]", r"^[\*]$");
toregex!(re9, "[+]", r"^[\+]$");
toregex!(re10, "+", r"^\+$");
toregex!(re11, "**", r"^.*$");
toregex!(re12, "☃", r"^\xe2\x98\x83$");
toregex!(re11, "☃", r"^\xe2\x98\x83$");
toregex!(re12, "**", r"^.*$");
toregex!(re13, "**/", r"^.*$");
toregex!(re14, "**/*", r"^(?:/?|.*/).*$");
toregex!(re15, "**/**", r"^.*$");
toregex!(re16, "**/**/*", r"^(?:/?|.*/).*$");
toregex!(re17, "**/**/**", r"^.*$");
toregex!(re18, "**/**/**/*", r"^(?:/?|.*/).*$");
toregex!(re19, "a/**", r"^a(?:/?|/.*)$");
toregex!(re20, "a/**/**", r"^a(?:/?|/.*)$");
toregex!(re21, "a/**/**/**", r"^a(?:/?|/.*)$");
toregex!(re22, "a/**/b", r"^a(?:/|/.*/)b$");
toregex!(re23, "a/**/**/b", r"^a(?:/|/.*/)b$");
toregex!(re24, "a/**/**/**/b", r"^a(?:/|/.*/)b$");
toregex!(re25, "**/b", r"^(?:/?|.*/)b$");
toregex!(re26, "**/**/b", r"^(?:/?|.*/)b$");
toregex!(re27, "**/**/**/b", r"^(?:/?|.*/)b$");
toregex!(re28, "a**", r"^a.*.*$");
toregex!(re29, "**a", r"^.*.*a$");
toregex!(re30, "a**b", r"^a.*.*b$");
toregex!(re31, "***", r"^.*.*.*$");
toregex!(re32, "/a**", r"^/a.*.*$");
toregex!(re33, "/**a", r"^/.*.*a$");
toregex!(re34, "/a**b", r"^/a.*.*b$");
matches!(match1, "a", "a");
matches!(match2, "a*b", "a_b");
@@ -1253,8 +1295,11 @@ mod tests {
matches!(matchpat4, "*hello.txt", "some\\path\\to\\hello.txt");
matches!(matchpat5, "*hello.txt", "/an/absolute/path/to/hello.txt");
matches!(matchpat6, "*some/path/to/hello.txt", "some/path/to/hello.txt");
matches!(matchpat7, "*some/path/to/hello.txt",
"a/bigger/some/path/to/hello.txt");
matches!(
matchpat7,
"*some/path/to/hello.txt",
"a/bigger/some/path/to/hello.txt"
);
matches!(matchescape, "_[[]_[]]_[?]_[*]_!_", "_[_]_?_*_!_");
@@ -1317,28 +1362,44 @@ mod tests {
nmatches!(matchnot15, "[!-]", "-");
nmatches!(matchnot16, "*hello.txt", "hello.txt-and-then-some");
nmatches!(matchnot17, "*hello.txt", "goodbye.txt");
nmatches!(matchnot18, "*some/path/to/hello.txt",
"some/path/to/hello.txt-and-then-some");
nmatches!(matchnot19, "*some/path/to/hello.txt",
"some/other/path/to/hello.txt");
nmatches!(
matchnot18,
"*some/path/to/hello.txt",
"some/path/to/hello.txt-and-then-some"
);
nmatches!(
matchnot19,
"*some/path/to/hello.txt",
"some/other/path/to/hello.txt"
);
nmatches!(matchnot20, "a", "foo/a");
nmatches!(matchnot21, "./foo", "foo");
nmatches!(matchnot22, "**/foo", "foofoo");
nmatches!(matchnot23, "**/foo/bar", "foofoo/bar");
nmatches!(matchnot24, "/*.c", "mozilla-sha1/sha1.c");
nmatches!(matchnot25, "*.c", "mozilla-sha1/sha1.c", SLASHLIT);
nmatches!(matchnot26, "**/m4/ltoptions.m4",
"csharp/src/packages/repositories.config", SLASHLIT);
nmatches!(
matchnot26,
"**/m4/ltoptions.m4",
"csharp/src/packages/repositories.config",
SLASHLIT
);
nmatches!(matchnot27, "a[^0-9]b", "a0b");
nmatches!(matchnot28, "a[^0-9]b", "a9b");
nmatches!(matchnot29, "[^-]", "-");
nmatches!(matchnot30, "some/*/needle.txt", "some/needle.txt");
nmatches!(
matchrec31,
"some/*/needle.txt", "some/one/two/needle.txt", SLASHLIT);
"some/*/needle.txt",
"some/one/two/needle.txt",
SLASHLIT
);
nmatches!(
matchrec32,
"some/*/needle.txt", "some/one/two/three/needle.txt", SLASHLIT);
"some/*/needle.txt",
"some/one/two/three/needle.txt",
SLASHLIT
);
macro_rules! extract {
($which:ident, $name:ident, $pat:expr, $expect:expr) => {
@@ -1400,19 +1461,27 @@ mod tests {
literal!(extract_lit7, "foo/bar", Some(s("foo/bar")));
literal!(extract_lit8, "**/foo/bar", None);
basetokens!(extract_basetoks1, "**/foo", Some(&*vec![
Literal('f'), Literal('o'), Literal('o'),
]));
basetokens!(
extract_basetoks1,
"**/foo",
Some(&*vec![Literal('f'), Literal('o'), Literal('o'),])
);
basetokens!(extract_basetoks2, "**/foo", None, CASEI);
basetokens!(extract_basetoks3, "**/foo", Some(&*vec![
Literal('f'), Literal('o'), Literal('o'),
]), SLASHLIT);
basetokens!(
extract_basetoks3,
"**/foo",
Some(&*vec![Literal('f'), Literal('o'), Literal('o'),]),
SLASHLIT
);
basetokens!(extract_basetoks4, "*foo", None, SLASHLIT);
basetokens!(extract_basetoks5, "*foo", None);
basetokens!(extract_basetoks6, "**/fo*o", None);
basetokens!(extract_basetoks7, "**/fo*o", Some(&*vec![
Literal('f'), Literal('o'), ZeroOrMore, Literal('o'),
]), SLASHLIT);
basetokens!(
extract_basetoks7,
"**/fo*o",
Some(&*vec![Literal('f'), Literal('o'), ZeroOrMore, Literal('o'),]),
SLASHLIT
);
ext!(extract_ext1, "**/*.rs", Some(s(".rs")));
ext!(extract_ext2, "**/*.rs.bak", None);

154
globset/src/lib.rs → crates/globset/src/lib.rs
View File

@@ -104,33 +104,37 @@ or to enable case insensitive matching.
#![deny(missing_docs)]
extern crate aho_corasick;
extern crate bstr;
extern crate fnv;
#[macro_use]
extern crate log;
extern crate memchr;
extern crate regex;
#[cfg(feature = "serde1")]
extern crate serde;
use std::borrow::Cow;
use std::collections::{BTreeMap, HashMap};
use std::error::Error as StdError;
use std::ffi::OsStr;
use std::fmt;
use std::hash;
use std::path::Path;
use std::str;
use aho_corasick::{Automaton, AcAutomaton, FullAcAutomaton};
use aho_corasick::AhoCorasick;
use bstr::{ByteSlice, ByteVec, B};
use regex::bytes::{Regex, RegexBuilder, RegexSet};
use pathutil::{
file_name, file_name_ext, normalize_path, os_str_bytes, path_bytes,
};
use glob::MatchStrategy;
pub use glob::{Glob, GlobBuilder, GlobMatcher};
use pathutil::{file_name, file_name_ext, normalize_path};
mod glob;
mod pathutil;
#[cfg(feature = "serde1")]
mod serde_impl;
/// Represents an error that can occur when parsing a glob pattern.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Error {
@@ -143,8 +147,13 @@ pub struct Error {
/// The kind of error that can occur when parsing a glob pattern.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ErrorKind {
/// Occurs when a use of `**` is invalid. Namely, `**` can only appear
/// adjacent to a path separator, or the beginning/end of a glob.
/// **DEPRECATED**.
///
/// This error used to occur for consistency with git's glob specification,
/// but the specification now accepts all uses of `**`. When `**` does not
/// appear adjacent to a path separator or at the beginning/end of a glob,
/// it is now treated as two consecutive `*` patterns. As such, this error
/// is no longer used.
InvalidRecursive,
/// Occurs when a character class (e.g., `[abc]`) is not closed.
UnclosedClass,
@@ -199,9 +208,7 @@ impl ErrorKind {
ErrorKind::UnclosedClass => {
"unclosed character class; missing ']'"
}
ErrorKind::InvalidRange(_, _) => {
"invalid character range"
}
ErrorKind::InvalidRange(_, _) => "invalid character range",
ErrorKind::UnopenedAlternates => {
"unopened alternate group; missing '{' \
(maybe escape '}' with '[}]'?)"
@@ -213,9 +220,7 @@ impl ErrorKind {
ErrorKind::NestedAlternates => {
"nested alternate groups are not allowed"
}
ErrorKind::DanglingEscape => {
"dangling '\\'"
}
ErrorKind::DanglingEscape => "dangling '\\'",
ErrorKind::Regex(ref err) => err,
ErrorKind::__Nonexhaustive => unreachable!(),
}
@@ -242,9 +247,7 @@ impl fmt::Display for ErrorKind {
| ErrorKind::UnclosedAlternates
| ErrorKind::NestedAlternates
| ErrorKind::DanglingEscape
| ErrorKind::Regex(_) => {
write!(f, "{}", self.description())
}
| ErrorKind::Regex(_) => write!(f, "{}", self.description()),
ErrorKind::InvalidRange(s, e) => {
write!(f, "invalid range; '{}' > '{}'", s, e)
}
@@ -259,21 +262,20 @@ fn new_regex(pat: &str) -> Result<Regex, Error> {
.size_limit(10 * (1 << 20))
.dfa_size_limit(10 * (1 << 20))
.build()
.map_err(|err| {
Error {
glob: Some(pat.to_string()),
kind: ErrorKind::Regex(err.to_string()),
}
.map_err(|err| Error {
glob: Some(pat.to_string()),
kind: ErrorKind::Regex(err.to_string()),
})
}
fn new_regex_set<I, S>(pats: I) -> Result<RegexSet, Error>
where S: AsRef<str>, I: IntoIterator<Item=S> {
RegexSet::new(pats).map_err(|err| {
Error {
glob: None,
kind: ErrorKind::Regex(err.to_string()),
}
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
RegexSet::new(pats).map_err(|err| Error {
glob: None,
kind: ErrorKind::Regex(err.to_string()),
})
}
@@ -289,19 +291,19 @@ pub struct GlobSet {
impl GlobSet {
/// Create an empty `GlobSet`. An empty set matches nothing.
#[inline]
pub fn empty() -> GlobSet {
GlobSet {
len: 0,
strats: vec![],
}
GlobSet { len: 0, strats: vec![] }
}
/// Returns true if this set is empty, and therefore matches nothing.
#[inline]
pub fn is_empty(&self) -> bool {
self.len == 0
}
/// Returns the number of globs in this set.
#[inline]
pub fn len(&self) -> usize {
self.len
}
@@ -426,11 +428,17 @@ impl GlobSet {
}
}
}
debug!("built glob set; {} literals, {} basenames, {} extensions, \
debug!(
"built glob set; {} literals, {} basenames, {} extensions, \
{} prefixes, {} suffixes, {} required extensions, {} regexes",
lits.0.len(), base_lits.0.len(), exts.0.len(),
prefixes.literals.len(), suffixes.literals.len(),
required_exts.0.len(), regexes.literals.len());
lits.0.len(),
base_lits.0.len(),
exts.0.len(),
prefixes.literals.len(),
suffixes.literals.len(),
required_exts.0.len(),
regexes.literals.len()
);
Ok(GlobSet {
len: pats.len(),
strats: vec![
@@ -440,7 +448,8 @@ impl GlobSet {
GlobSetMatchStrategy::Suffix(suffixes.suffix()),
GlobSetMatchStrategy::Prefix(prefixes.prefix()),
GlobSetMatchStrategy::RequiredExtension(
required_exts.build()?),
required_exts.build()?,
),
GlobSetMatchStrategy::Regex(regexes.regex_set()?),
],
})
@@ -492,13 +501,10 @@ pub struct Candidate<'a> {
impl<'a> Candidate<'a> {
/// Create a new candidate for matching from the given path.
pub fn new<P: AsRef<Path> + ?Sized>(path: &'a P) -> Candidate<'a> {
let path = path.as_ref();
let basename = file_name(path).unwrap_or(OsStr::new(""));
Candidate {
path: normalize_path(path_bytes(path)),
basename: os_str_bytes(basename),
ext: file_name_ext(basename).unwrap_or(Cow::Borrowed(b"")),
}
let path = normalize_path(Vec::from_path_lossy(path.as_ref()));
let basename = file_name(&path).unwrap_or(Cow::Borrowed(B("")));
let ext = file_name_ext(&basename).unwrap_or(Cow::Borrowed(B("")));
Candidate { path: path, basename: basename, ext: ext }
}
fn path_prefix(&self, max: usize) -> &[u8] {
@@ -570,12 +576,12 @@ impl LiteralStrategy {
}
fn is_match(&self, candidate: &Candidate) -> bool {
self.0.contains_key(&*candidate.path)
self.0.contains_key(candidate.path.as_bytes())
}
#[inline(never)]
fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {
if let Some(hits) = self.0.get(&*candidate.path) {
if let Some(hits) = self.0.get(candidate.path.as_bytes()) {
matches.extend(hits);
}
}
@@ -597,7 +603,7 @@ impl BasenameLiteralStrategy {
if candidate.basename.is_empty() {
return false;
}
self.0.contains_key(&*candidate.basename)
self.0.contains_key(candidate.basename.as_bytes())
}
#[inline(never)]
@@ -605,7 +611,7 @@ impl BasenameLiteralStrategy {
if candidate.basename.is_empty() {
return;
}
if let Some(hits) = self.0.get(&*candidate.basename) {
if let Some(hits) = self.0.get(candidate.basename.as_bytes()) {
matches.extend(hits);
}
}
@@ -627,7 +633,7 @@ impl ExtensionStrategy {
if candidate.ext.is_empty() {
return false;
}
self.0.contains_key(&*candidate.ext)
self.0.contains_key(candidate.ext.as_bytes())
}
#[inline(never)]
@@ -635,7 +641,7 @@ impl ExtensionStrategy {
if candidate.ext.is_empty() {
return;
}
if let Some(hits) = self.0.get(&*candidate.ext) {
if let Some(hits) = self.0.get(candidate.ext.as_bytes()) {
matches.extend(hits);
}
}
@@ -643,7 +649,7 @@ impl ExtensionStrategy {
#[derive(Clone, Debug)]
struct PrefixStrategy {
matcher: FullAcAutomaton<Vec<u8>>,
matcher: AhoCorasick,
map: Vec<usize>,
longest: usize,
}
@@ -651,8 +657,8 @@ struct PrefixStrategy {
impl PrefixStrategy {
fn is_match(&self, candidate: &Candidate) -> bool {
let path = candidate.path_prefix(self.longest);
for m in self.matcher.find_overlapping(path) {
if m.start == 0 {
for m in self.matcher.find_overlapping_iter(path) {
if m.start() == 0 {
return true;
}
}
@@ -661,9 +667,9 @@ impl PrefixStrategy {
fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {
let path = candidate.path_prefix(self.longest);
for m in self.matcher.find_overlapping(path) {
if m.start == 0 {
matches.push(self.map[m.pati]);
for m in self.matcher.find_overlapping_iter(path) {
if m.start() == 0 {
matches.push(self.map[m.pattern()]);
}
}
}
@@ -671,7 +677,7 @@ impl PrefixStrategy {
#[derive(Clone, Debug)]
struct SuffixStrategy {
matcher: FullAcAutomaton<Vec<u8>>,
matcher: AhoCorasick,
map: Vec<usize>,
longest: usize,
}
@@ -679,8 +685,8 @@ struct SuffixStrategy {
impl SuffixStrategy {
fn is_match(&self, candidate: &Candidate) -> bool {
let path = candidate.path_suffix(self.longest);
for m in self.matcher.find_overlapping(path) {
if m.end == path.len() {
for m in self.matcher.find_overlapping_iter(path) {
if m.end() == path.len() {
return true;
}
}
@@ -689,9 +695,9 @@ impl SuffixStrategy {
fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {
let path = candidate.path_suffix(self.longest);
for m in self.matcher.find_overlapping(path) {
if m.end == path.len() {
matches.push(self.map[m.pati]);
for m in self.matcher.find_overlapping_iter(path) {
if m.end() == path.len() {
matches.push(self.map[m.pattern()]);
}
}
}
@@ -705,11 +711,11 @@ impl RequiredExtensionStrategy {
if candidate.ext.is_empty() {
return false;
}
match self.0.get(&*candidate.ext) {
match self.0.get(candidate.ext.as_bytes()) {
None => false,
Some(regexes) => {
for &(_, ref re) in regexes {
if re.is_match(&*candidate.path) {
if re.is_match(candidate.path.as_bytes()) {
return true;
}
}
@@ -723,9 +729,9 @@ impl RequiredExtensionStrategy {
if candidate.ext.is_empty() {
return;
}
if let Some(regexes) = self.0.get(&*candidate.ext) {
if let Some(regexes) = self.0.get(candidate.ext.as_bytes()) {
for &(global_index, ref re) in regexes {
if re.is_match(&*candidate.path) {
if re.is_match(candidate.path.as_bytes()) {
matches.push(global_index);
}
}
@@ -741,11 +747,11 @@ struct RegexSetStrategy {
impl RegexSetStrategy {
fn is_match(&self, candidate: &Candidate) -> bool {
self.matcher.is_match(&*candidate.path)
self.matcher.is_match(candidate.path.as_bytes())
}
fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {
for i in self.matcher.matches(&*candidate.path) {
for i in self.matcher.matches(candidate.path.as_bytes()) {
matches.push(self.map[i]);
}
}
@@ -760,11 +766,7 @@ struct MultiStrategyBuilder {
impl MultiStrategyBuilder {
fn new() -> MultiStrategyBuilder {
MultiStrategyBuilder {
literals: vec![],
map: vec![],
longest: 0,
}
MultiStrategyBuilder { literals: vec![], map: vec![], longest: 0 }
}
fn add(&mut self, global_index: usize, literal: String) {
@@ -776,18 +778,16 @@ impl MultiStrategyBuilder {
}
fn prefix(self) -> PrefixStrategy {
let it = self.literals.into_iter().map(|s| s.into_bytes());
PrefixStrategy {
matcher: AcAutomaton::new(it).into_full(),
matcher: AhoCorasick::new_auto_configured(&self.literals),
map: self.map,
longest: self.longest,
}
}
fn suffix(self) -> SuffixStrategy {
let it = self.literals.into_iter().map(|s| s.into_bytes());
SuffixStrategy {
matcher: AcAutomaton::new(it).into_full(),
matcher: AhoCorasick::new_auto_configured(&self.literals),
map: self.map,
longest: self.longest,
}

99
globset/src/pathutil.rs → crates/globset/src/pathutil.rs
View File

@@ -1,41 +1,26 @@
use std::borrow::Cow;
use std::ffi::OsStr;
use std::path::Path;
use bstr::{ByteSlice, ByteVec};
/// The final component of the path, if it is a normal file.
///
/// If the path terminates in ., .., or consists solely of a root of prefix,
/// file_name will return None.
#[cfg(unix)]
pub fn file_name<'a, P: AsRef<Path> + ?Sized>(
path: &'a P,
) -> Option<&'a OsStr> {
use std::os::unix::ffi::OsStrExt;
use memchr::memrchr;
let path = path.as_ref().as_os_str().as_bytes();
pub fn file_name<'a>(path: &Cow<'a, [u8]>) -> Option<Cow<'a, [u8]>> {
if path.is_empty() {
return None;
} else if path.len() == 1 && path[0] == b'.' {
return None;
} else if path.last() == Some(&b'.') {
return None;
} else if path.len() >= 2 && &path[path.len() - 2..] == &b".."[..] {
} else if path.last_byte() == Some(b'.') {
return None;
}
let last_slash = memrchr(b'/', path).map(|i| i + 1).unwrap_or(0);
Some(OsStr::from_bytes(&path[last_slash..]))
}
/// The final component of the path, if it is a normal file.
///
/// If the path terminates in ., .., or consists solely of a root of prefix,
/// file_name will return None.
#[cfg(not(unix))]
pub fn file_name<'a, P: AsRef<Path> + ?Sized>(
path: &'a P,
) -> Option<&'a OsStr> {
path.as_ref().file_name()
let last_slash = path.rfind_byte(b'/').map(|i| i + 1).unwrap_or(0);
Some(match *path {
Cow::Borrowed(path) => Cow::Borrowed(&path[last_slash..]),
Cow::Owned(ref path) => {
let mut path = path.clone();
path.drain_bytes(..last_slash);
Cow::Owned(path)
}
})
}
/// Return a file extension given a path's file name.
@@ -54,55 +39,24 @@ pub fn file_name<'a, P: AsRef<Path> + ?Sized>(
/// a pattern like `*.rs` is obviously trying to match files with a `rs`
/// extension, but it also matches files like `.rs`, which doesn't have an
/// extension according to std::path::Path::extension.
pub fn file_name_ext(name: &OsStr) -> Option<Cow<[u8]>> {
pub fn file_name_ext<'a>(name: &Cow<'a, [u8]>) -> Option<Cow<'a, [u8]>> {
if name.is_empty() {
return None;
}
let name = os_str_bytes(name);
let last_dot_at = {
let result = name
.iter().enumerate().rev()
.find(|&(_, &b)| b == b'.')
.map(|(i, _)| i);
match result {
None => return None,
Some(i) => i,
}
let last_dot_at = match name.rfind_byte(b'.') {
None => return None,
Some(i) => i,
};
Some(match name {
Some(match *name {
Cow::Borrowed(name) => Cow::Borrowed(&name[last_dot_at..]),
Cow::Owned(mut name) => {
name.drain(..last_dot_at);
Cow::Owned(ref name) => {
let mut name = name.clone();
name.drain_bytes(..last_dot_at);
Cow::Owned(name)
}
})
}
/// Return raw bytes of a path, transcoded to UTF-8 if necessary.
pub fn path_bytes(path: &Path) -> Cow<[u8]> {
os_str_bytes(path.as_os_str())
}
/// Return the raw bytes of the given OS string, possibly transcoded to UTF-8.
#[cfg(unix)]
pub fn os_str_bytes(s: &OsStr) -> Cow<[u8]> {
use std::os::unix::ffi::OsStrExt;
Cow::Borrowed(s.as_bytes())
}
/// Return the raw bytes of the given OS string, possibly transcoded to UTF-8.
#[cfg(not(unix))]
pub fn os_str_bytes(s: &OsStr) -> Cow<[u8]> {
// TODO(burntsushi): On Windows, OS strings are WTF-8, which is a superset
// of UTF-8, so even if we could get at the raw bytes, they wouldn't
// be useful. We *must* convert to UTF-8 before doing path matching.
// Unfortunate, but necessary.
match s.to_string_lossy() {
Cow::Owned(s) => Cow::Owned(s.into_bytes()),
Cow::Borrowed(s) => Cow::Borrowed(s.as_bytes()),
}
}
/// Normalizes a path to use `/` as a separator everywhere, even on platforms
/// that recognize other characters as separators.
#[cfg(unix)]
@@ -129,7 +83,8 @@ pub fn normalize_path(mut path: Cow<[u8]>) -> Cow<[u8]> {
#[cfg(test)]
mod tests {
use std::borrow::Cow;
use std::ffi::OsStr;
use bstr::{ByteVec, B};
use super::{file_name_ext, normalize_path};
@@ -137,8 +92,9 @@ mod tests {
($name:ident, $file_name:expr, $ext:expr) => {
#[test]
fn $name() {
let got = file_name_ext(OsStr::new($file_name));
assert_eq!($ext.map(|s| Cow::Borrowed(s.as_bytes())), got);
let bs = Vec::from($file_name);
let got = file_name_ext(&Cow::Owned(bs));
assert_eq!($ext.map(|s| Cow::Borrowed(B(s))), got);
}
};
}
@@ -153,7 +109,8 @@ mod tests {
($name:ident, $path:expr, $expected:expr) => {
#[test]
fn $name() {
let got = normalize_path(Cow::Owned($path.to_vec()));
let bs = Vec::from_slice($path);
let got = normalize_path(Cow::Owned(bs));
assert_eq!($expected.to_vec(), got.into_owned());
}
};

38
crates/globset/src/serde_impl.rs Normal file
View File

@@ -0,0 +1,38 @@
use serde::de::Error;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use Glob;
impl Serialize for Glob {
fn serialize<S: Serializer>(
&self,
serializer: S,
) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.glob())
}
}
impl<'de> Deserialize<'de> for Glob {
fn deserialize<D: Deserializer<'de>>(
deserializer: D,
) -> Result<Self, D::Error> {
let glob = <&str as Deserialize>::deserialize(deserializer)?;
Glob::new(glob).map_err(D::Error::custom)
}
}
#[cfg(test)]
mod tests {
use Glob;
#[test]
fn glob_json_works() {
let test_glob = Glob::new("src/**/*.rs").unwrap();
let ser = serde_json::to_string(&test_glob).unwrap();
assert_eq!(ser, "\"src/**/*.rs\"");
let de: Glob = serde_json::from_str(&ser).unwrap();
assert_eq!(test_glob, de);
}
}

0
grep/COPYING → crates/grep/COPYING
View File

32
crates/grep/Cargo.toml Normal file
View File

@@ -0,0 +1,32 @@
[package]
name = "grep"
version = "0.2.7" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """
Fast line oriented regex searching as a library.
"""
documentation = "http://burntsushi.net/rustdoc/grep/"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/grep"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/grep"
readme = "README.md"
keywords = ["regex", "grep", "egrep", "search", "pattern"]
license = "Unlicense/MIT"
[dependencies]
grep-cli = { version = "0.1.5", path = "../cli" }
grep-matcher = { version = "0.1.4", path = "../matcher" }
grep-pcre2 = { version = "0.1.4", path = "../pcre2", optional = true }
grep-printer = { version = "0.1.5", path = "../printer" }
grep-regex = { version = "0.1.8", path = "../regex" }
grep-searcher = { version = "0.1.7", path = "../searcher" }
[dev-dependencies]
termcolor = "1.0.4"
walkdir = "2.2.7"
[features]
simd-accel = ["grep-searcher/simd-accel"]
pcre2 = ["grep-pcre2"]
# This feature is DEPRECATED. Runtime dispatch is used for SIMD now.
avx-accel = []

0
grep-matcher/LICENSE-MIT → crates/grep/LICENSE-MIT
View File

0
grep/README.md → crates/grep/README.md
View File

0
grep-matcher/UNLICENSE → crates/grep/UNLICENSE
View File

16
grep/examples/simplegrep.rs → crates/grep/examples/simplegrep.rs
View File

@@ -21,7 +21,7 @@ fn main() {
}
}
fn try_main() -> Result<(), Box<Error>> {
fn try_main() -> Result<(), Box<dyn Error>> {
let mut args: Vec<OsString> = env::args_os().collect();
if args.len() < 2 {
return Err("Usage: simplegrep <pattern> [<path> ...]".into());
@@ -32,7 +32,7 @@ fn try_main() -> Result<(), Box<Error>> {
search(cli::pattern_from_os(&args[1])?, &args[2..])
}
fn search(pattern: &str, paths: &[OsString]) -> Result<(), Box<Error>> {
fn search(pattern: &str, paths: &[OsString]) -> Result<(), Box<dyn Error>> {
let matcher = RegexMatcher::new_line_matcher(&pattern)?;
let mut searcher = SearcherBuilder::new()
.binary_detection(BinaryDetection::quit(b'\x00'))
@@ -40,13 +40,11 @@ fn search(pattern: &str, paths: &[OsString]) -> Result<(), Box<Error>> {
.build();
let mut printer = StandardBuilder::new()
.color_specs(ColorSpecs::default_with_color())
.build(cli::stdout(
if cli::is_tty_stdout() {
ColorChoice::Auto
} else {
ColorChoice::Never
}
));
.build(cli::stdout(if cli::is_tty_stdout() {
ColorChoice::Auto
} else {
ColorChoice::Never
}));
for path in paths {
for result in WalkDir::new(path) {

0
grep/src/lib.rs → crates/grep/src/lib.rs
View File

0
ignore/COPYING → crates/ignore/COPYING
View File

26
ignore/Cargo.toml → crates/ignore/Cargo.toml
View File

@@ -1,14 +1,14 @@
[package]
name = "ignore"
version = "0.4.4" #:version
version = "0.4.16" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """
A fast library for efficiently matching ignore files such as `.gitignore`
against file paths.
"""
documentation = "https://docs.rs/ignore"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/ignore"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/ignore"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/ignore"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/ignore"
readme = "README.md"
keywords = ["glob", "ignore", "gitignore", "pattern", "file"]
license = "Unlicense/MIT"
@@ -18,21 +18,21 @@ name = "ignore"
bench = false
[dependencies]
crossbeam-channel = "0.2.4"
globset = { version = "0.4.2", path = "../globset" }
lazy_static = "1.1.0"
crossbeam-utils = "0.7.0"
globset = { version = "0.4.5", path = "../globset" }
lazy_static = "1.1"
log = "0.4.5"
memchr = "2.0.2"
regex = "1.0.5"
same-file = "1.0.3"
thread_local = "0.3.6"
walkdir = "2.2.5"
memchr = "2.1"
regex = "1.1"
same-file = "1.0.4"
thread_local = "1"
walkdir = "2.2.7"
[target.'cfg(windows)'.dependencies.winapi-util]
version = "0.1.1"
version = "0.1.2"
[dev-dependencies]
tempdir = "0.3.7"
crossbeam-channel = "0.4.0"
[features]
simd-accel = ["globset/simd-accel"]

0
grep-pcre2/LICENSE-MIT → crates/ignore/LICENSE-MIT
View File

0
ignore/README.md → crates/ignore/README.md
View File

0
grep-pcre2/UNLICENSE → crates/ignore/UNLICENSE
View File

6
ignore/examples/walk.rs → crates/ignore/examples/walk.rs
View File

@@ -37,19 +37,19 @@ fn main() {
Box::new(move |result| {
use ignore::WalkState::*;
tx.send(DirEntry::Y(result.unwrap()));
tx.send(DirEntry::Y(result.unwrap())).unwrap();
Continue
})
});
} else if simple {
let walker = WalkDir::new(path);
for result in walker {
tx.send(DirEntry::X(result.unwrap()));
tx.send(DirEntry::X(result.unwrap())).unwrap();
}
} else {
let walker = WalkBuilder::new(path).build();
for result in walker {
tx.send(DirEntry::Y(result.unwrap()));
tx.send(DirEntry::Y(result.unwrap())).unwrap();
}
}
drop(tx);

249
crates/ignore/src/default_types.rs Normal file
View File

@@ -0,0 +1,249 @@
/// This list represents the default file types that ripgrep ships with. In
/// general, any file format is fair game, although it should generally be
/// limited to reasonably popular open formats. For other cases, you can add
/// types to each invocation of ripgrep with the '--type-add' flag.
///
/// If you would like to add or improve this list, please file a PR:
/// https://github.com/BurntSushi/ripgrep
///
/// Please try to keep this list sorted lexicographically and wrapped to 79
/// columns (inclusive).
#[rustfmt::skip]
pub const DEFAULT_TYPES: &[(&str, &[&str])] = &[
("agda", &["*.agda", "*.lagda"]),
("aidl", &["*.aidl"]),
("amake", &["*.mk", "*.bp"]),
("asciidoc", &["*.adoc", "*.asc", "*.asciidoc"]),
("asm", &["*.asm", "*.s", "*.S"]),
("asp", &[
"*.aspx", "*.aspx.cs", "*.aspx.cs", "*.ascx", "*.ascx.cs", "*.ascx.vb",
]),
("ats", &["*.ats", "*.dats", "*.sats", "*.hats"]),
("avro", &["*.avdl", "*.avpr", "*.avsc"]),
("awk", &["*.awk"]),
("bazel", &["*.bzl", "WORKSPACE", "BUILD", "BUILD.bazel"]),
("bitbake", &["*.bb", "*.bbappend", "*.bbclass", "*.conf", "*.inc"]),
("brotli", &["*.br"]),
("buildstream", &["*.bst"]),
("bzip2", &["*.bz2", "*.tbz2"]),
("c", &["*.[chH]", "*.[chH].in", "*.cats"]),
("cabal", &["*.cabal"]),
("cbor", &["*.cbor"]),
("ceylon", &["*.ceylon"]),
("clojure", &["*.clj", "*.cljc", "*.cljs", "*.cljx"]),
("cmake", &["*.cmake", "CMakeLists.txt"]),
("coffeescript", &["*.coffee"]),
("config", &["*.cfg", "*.conf", "*.config", "*.ini"]),
("coq", &["*.v"]),
("cpp", &[
"*.[ChH]", "*.cc", "*.[ch]pp", "*.[ch]xx", "*.hh", "*.inl",
"*.[ChH].in", "*.cc.in", "*.[ch]pp.in", "*.[ch]xx.in", "*.hh.in",
]),
("creole", &["*.creole"]),
("crystal", &["Projectfile", "*.cr"]),
("cs", &["*.cs"]),
("csharp", &["*.cs"]),
("cshtml", &["*.cshtml"]),
("css", &["*.css", "*.scss"]),
("csv", &["*.csv"]),
("cython", &["*.pyx", "*.pxi", "*.pxd"]),
("d", &["*.d"]),
("dart", &["*.dart"]),
("dhall", &["*.dhall"]),
("diff", &["*.patch", "*.diff"]),
("docker", &["*Dockerfile*"]),
("ebuild", &["*.ebuild"]),
("edn", &["*.edn"]),
("elisp", &["*.el"]),
("elixir", &["*.ex", "*.eex", "*.exs"]),
("elm", &["*.elm"]),
("erb", &["*.erb"]),
("erlang", &["*.erl", "*.hrl"]),
("fidl", &["*.fidl"]),
("fish", &["*.fish"]),
("fortran", &[
"*.f", "*.F", "*.f77", "*.F77", "*.pfo",
"*.f90", "*.F90", "*.f95", "*.F95",
]),
("fsharp", &["*.fs", "*.fsx", "*.fsi"]),
("gap", &["*.g", "*.gap", "*.gi", "*.gd", "*.tst"]),
("gn", &["*.gn", "*.gni"]),
("go", &["*.go"]),
("gradle", &["*.gradle"]),
("groovy", &["*.groovy", "*.gradle"]),
("gzip", &["*.gz", "*.tgz"]),
("h", &["*.h", "*.hpp"]),
("haml", &["*.haml"]),
("haskell", &["*.hs", "*.lhs", "*.cpphs", "*.c2hs", "*.hsc"]),
("hbs", &["*.hbs"]),
("hs", &["*.hs", "*.lhs"]),
("html", &["*.htm", "*.html", "*.ejs"]),
("idris", &["*.idr", "*.lidr"]),
("java", &["*.java", "*.jsp", "*.jspx", "*.properties"]),
("jinja", &["*.j2", "*.jinja", "*.jinja2"]),
("jl", &["*.jl"]),
("js", &["*.js", "*.jsx", "*.vue"]),
("json", &["*.json", "composer.lock"]),
("jsonl", &["*.jsonl"]),
("julia", &["*.jl"]),
("jupyter", &["*.ipynb", "*.jpynb"]),
("k", &["*.k"]),
("kotlin", &["*.kt", "*.kts"]),
("less", &["*.less"]),
("license", &[
// General
"COPYING", "COPYING[.-]*",
"COPYRIGHT", "COPYRIGHT[.-]*",
"EULA", "EULA[.-]*",
"licen[cs]e", "licen[cs]e.*",
"LICEN[CS]E", "LICEN[CS]E[.-]*", "*[.-]LICEN[CS]E*",
"NOTICE", "NOTICE[.-]*",
"PATENTS", "PATENTS[.-]*",
"UNLICEN[CS]E", "UNLICEN[CS]E[.-]*",
// GPL (gpl.txt, etc.)
"agpl[.-]*",
"gpl[.-]*",
"lgpl[.-]*",
// Other license-specific (APACHE-2.0.txt, etc.)
"AGPL-*[0-9]*",
"APACHE-*[0-9]*",
"BSD-*[0-9]*",
"CC-BY-*",
"GFDL-*[0-9]*",
"GNU-*[0-9]*",
"GPL-*[0-9]*",
"LGPL-*[0-9]*",
"MIT-*[0-9]*",
"MPL-*[0-9]*",
"OFL-*[0-9]*",
]),
("lisp", &["*.el", "*.jl", "*.lisp", "*.lsp", "*.sc", "*.scm"]),
("lock", &["*.lock", "package-lock.json"]),
("log", &["*.log"]),
("lua", &["*.lua"]),
("lz4", &["*.lz4"]),
("lzma", &["*.lzma"]),
("m4", &["*.ac", "*.m4"]),
("make", &[
"[Gg][Nn][Uu]makefile", "[Mm]akefile",
"[Gg][Nn][Uu]makefile.am", "[Mm]akefile.am",
"[Gg][Nn][Uu]makefile.in", "[Mm]akefile.in",
"*.mk", "*.mak"
]),
("mako", &["*.mako", "*.mao"]),
("man", &["*.[0-9lnpx]", "*.[0-9][cEFMmpSx]"]),
("markdown", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
("matlab", &["*.m"]),
("md", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
("meson", &["meson.build", "meson_options.txt"]),
("mk", &["mkfile"]),
("ml", &["*.ml"]),
("msbuild", &[
"*.csproj", "*.fsproj", "*.vcxproj", "*.proj", "*.props", "*.targets",
]),
("nim", &["*.nim", "*.nimf", "*.nimble", "*.nims"]),
("nix", &["*.nix"]),
("objc", &["*.h", "*.m"]),
("objcpp", &["*.h", "*.mm"]),
("ocaml", &["*.ml", "*.mli", "*.mll", "*.mly"]),
("org", &["*.org", "*.org_archive"]),
("pascal", &["*.pas", "*.dpr", "*.lpr", "*.pp", "*.inc"]),
("pdf", &["*.pdf"]),
("perl", &["*.perl", "*.pl", "*.PL", "*.plh", "*.plx", "*.pm", "*.t"]),
("php", &["*.php", "*.php3", "*.php4", "*.php5", "*.phtml"]),
("pod", &["*.pod"]),
("postscript", &["*.eps", "*.ps"]),
("protobuf", &["*.proto"]),
("ps", &["*.cdxml", "*.ps1", "*.ps1xml", "*.psd1", "*.psm1"]),
("puppet", &["*.erb", "*.pp", "*.rb"]),
("purs", &["*.purs"]),
("py", &["*.py"]),
("qmake", &["*.pro", "*.pri", "*.prf"]),
("qml", &["*.qml"]),
("r", &["*.R", "*.r", "*.Rmd", "*.Rnw"]),
("rdoc", &["*.rdoc"]),
("readme", &["README*", "*README"]),
("robot", &["*.robot"]),
("rst", &["*.rst"]),
("ruby", &["Gemfile", "*.gemspec", ".irbrc", "Rakefile", "*.rb"]),
("rust", &["*.rs"]),
("sass", &["*.sass", "*.scss"]),
("scala", &["*.scala", "*.sbt"]),
("sh", &[
// Portable/misc. init files
".login", ".logout", ".profile", "profile",
// bash-specific init files
".bash_login", "bash_login",
".bash_logout", "bash_logout",
".bash_profile", "bash_profile",
".bashrc", "bashrc", "*.bashrc",
// csh-specific init files
".cshrc", "*.cshrc",
// ksh-specific init files
".kshrc", "*.kshrc",
// tcsh-specific init files
".tcshrc",
// zsh-specific init files
".zshenv", "zshenv",
".zlogin", "zlogin",
".zlogout", "zlogout",
".zprofile", "zprofile",
".zshrc", "zshrc",
// Extensions
"*.bash", "*.csh", "*.ksh", "*.sh", "*.tcsh", "*.zsh",
]),
("slim", &["*.skim", "*.slim", "*.slime"]),
("smarty", &["*.tpl"]),
("sml", &["*.sml", "*.sig"]),
("soy", &["*.soy"]),
("spark", &["*.spark"]),
("spec", &["*.spec"]),
("sql", &["*.sql", "*.psql"]),
("stylus", &["*.styl"]),
("sv", &["*.v", "*.vg", "*.sv", "*.svh", "*.h"]),
("svg", &["*.svg"]),
("swift", &["*.swift"]),
("swig", &["*.def", "*.i"]),
("systemd", &[
"*.automount", "*.conf", "*.device", "*.link", "*.mount", "*.path",
"*.scope", "*.service", "*.slice", "*.socket", "*.swap", "*.target",
"*.timer",
]),
("taskpaper", &["*.taskpaper"]),
("tcl", &["*.tcl"]),
("tex", &["*.tex", "*.ltx", "*.cls", "*.sty", "*.bib", "*.dtx", "*.ins"]),
("textile", &["*.textile"]),
("tf", &["*.tf"]),
("thrift", &["*.thrift"]),
("toml", &["*.toml", "Cargo.lock"]),
("ts", &["*.ts", "*.tsx"]),
("twig", &["*.twig"]),
("txt", &["*.txt"]),
("typoscript", &["*.typoscript", "*.ts"]),
("vala", &["*.vala"]),
("vb", &["*.vb"]),
("verilog", &["*.v", "*.vh", "*.sv", "*.svh"]),
("vhdl", &["*.vhd", "*.vhdl"]),
("vim", &["*.vim"]),
("vimscript", &["*.vim"]),
("webidl", &["*.idl", "*.webidl", "*.widl"]),
("wiki", &["*.mediawiki", "*.wiki"]),
("xml", &[
"*.xml", "*.xml.dist", "*.dtd", "*.xsl", "*.xslt", "*.xsd", "*.xjb",
"*.rng", "*.sch", "*.xhtml",
]),
("xz", &["*.xz", "*.txz"]),
("yacc", &["*.y"]),
("yaml", &["*.yaml", "*.yml"]),
("z", &["*.Z"]),
("zig", &["*.zig"]),
("zsh", &[
".zshenv", "zshenv",
".zlogin", "zlogin",
".zlogout", "zlogout",
".zprofile", "zprofile",
".zshrc", "zshrc",
"*.zsh",
]),
("zstd", &["*.zst", "*.zstd"]),
];

491
ignore/src/dir.rs → crates/ignore/src/dir.rs
View File

@@ -14,14 +14,17 @@
// well.
use std::collections::HashMap;
use std::ffi::{OsString, OsStr};
use std::ffi::{OsStr, OsString};
use std::fs::{File, FileType};
use std::io::{self, BufRead};
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use gitignore::{self, Gitignore, GitignoreBuilder};
use pathutil::{is_hidden, strip_prefix};
use overrides::{self, Override};
use pathutil::{is_hidden, strip_prefix};
use types::{self, Types};
use walk::DirEntry;
use {Error, Match, PartialErrorBuilder};
/// IgnoreMatch represents information about where a match came from when using
@@ -73,6 +76,11 @@ struct IgnoreOptions {
git_ignore: bool,
/// Whether to read .git/info/exclude files.
git_exclude: bool,
/// Whether to ignore files case insensitively
ignore_case_insensitive: bool,
/// Whether a git repository must be present in order to apply any
/// git-related ignore rules.
require_git: bool,
}
/// Ignore is a matcher useful for recursively walking one or more directories.
@@ -194,7 +202,11 @@ impl Ignore {
errs.maybe_push(err);
igtmp.is_absolute_parent = true;
igtmp.absolute_base = Some(absolute_base.clone());
igtmp.has_git = parent.join(".git").exists();
igtmp.has_git = if self.0.opts.git_ignore {
parent.join(".git").exists()
} else {
false
};
ig = Ignore(Arc::new(igtmp));
compiled.insert(parent.as_os_str().to_os_string(), ig.clone());
}
@@ -219,40 +231,70 @@ impl Ignore {
/// Like add_child, but takes a full path and returns an IgnoreInner.
fn add_child_path(&self, dir: &Path) -> (IgnoreInner, Option<Error>) {
let git_type = if self.0.opts.git_ignore || self.0.opts.git_exclude {
dir.join(".git").metadata().ok().map(|md| md.file_type())
} else {
None
};
let has_git = git_type.map(|_| true).unwrap_or(false);
let mut errs = PartialErrorBuilder::default();
let custom_ig_matcher =
if self.0.custom_ignore_filenames.is_empty() {
Gitignore::empty()
} else {
let (m, err) =
create_gitignore(&dir, &self.0.custom_ignore_filenames);
errs.maybe_push(err);
m
};
let ig_matcher =
if !self.0.opts.ignore {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(&dir, &[".ignore"]);
errs.maybe_push(err);
m
};
let gi_matcher =
if !self.0.opts.git_ignore {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(&dir, &[".gitignore"]);
errs.maybe_push(err);
m
};
let gi_exclude_matcher =
if !self.0.opts.git_exclude {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(&dir, &[".git/info/exclude"]);
errs.maybe_push(err);
m
};
let custom_ig_matcher = if self.0.custom_ignore_filenames.is_empty() {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(
&dir,
&dir,
&self.0.custom_ignore_filenames,
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let ig_matcher = if !self.0.opts.ignore {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(
&dir,
&dir,
&[".ignore"],
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let gi_matcher = if !self.0.opts.git_ignore {
Gitignore::empty()
} else {
let (m, err) = create_gitignore(
&dir,
&dir,
&[".gitignore"],
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
};
let gi_exclude_matcher = if !self.0.opts.git_exclude {
Gitignore::empty()
} else {
match resolve_git_commondir(dir, git_type) {
Ok(git_dir) => {
let (m, err) = create_gitignore(
&dir,
&git_dir,
&["info/exclude"],
self.0.opts.ignore_case_insensitive,
);
errs.maybe_push(err);
m
}
Err(err) => {
errs.maybe_push(err);
Gitignore::empty()
}
}
};
let ig = IgnoreInner {
compiled: self.0.compiled.clone(),
dir: dir.to_path_buf(),
@@ -268,7 +310,7 @@ impl Ignore {
git_global_matcher: self.0.git_global_matcher.clone(),
git_ignore_matcher: gi_matcher,
git_exclude_matcher: gi_exclude_matcher,
has_git: dir.join(".git").exists(),
has_git,
opts: self.0.opts,
};
(ig, errs.into_error_option())
@@ -277,19 +319,35 @@ impl Ignore {
/// Returns true if at least one type of ignore rule should be matched.
fn has_any_ignore_rules(&self) -> bool {
let opts = self.0.opts;
let has_custom_ignore_files = !self.0.custom_ignore_filenames.is_empty();
let has_custom_ignore_files =
!self.0.custom_ignore_filenames.is_empty();
let has_explicit_ignores = !self.0.explicit_ignores.is_empty();
opts.ignore || opts.git_global || opts.git_ignore
|| opts.git_exclude || has_custom_ignore_files
|| has_explicit_ignores
opts.ignore
|| opts.git_global
|| opts.git_ignore
|| opts.git_exclude
|| has_custom_ignore_files
|| has_explicit_ignores
}
/// Like `matched`, but works with a directory entry instead.
pub fn matched_dir_entry<'a>(
&'a self,
dent: &DirEntry,
) -> Match<IgnoreMatch<'a>> {
let m = self.matched(dent.path(), dent.is_dir());
if m.is_none() && self.0.opts.hidden && is_hidden(dent) {
return Match::Ignore(IgnoreMatch::hidden());
}
m
}
/// Returns a match indicating whether the given file path should be
/// ignored or not.
///
/// The match contains information about its origin.
pub fn matched<'a, P: AsRef<Path>>(
fn matched<'a, P: AsRef<Path>>(
&'a self,
path: P,
is_dir: bool,
@@ -305,9 +363,11 @@ impl Ignore {
// return that result immediately. Overrides have the highest
// precedence.
if !self.0.overrides.is_empty() {
let mat =
self.0.overrides.matched(path, is_dir)
.map(IgnoreMatch::overrides);
let mat = self
.0
.overrides
.matched(path, is_dir)
.map(IgnoreMatch::overrides);
if !mat.is_none() {
return mat;
}
@@ -330,9 +390,6 @@ impl Ignore {
whitelisted = mat;
}
}
if whitelisted.is_none() && self.0.opts.hidden && is_hidden(path) {
return Match::Ignore(IgnoreMatch::hidden());
}
whitelisted
}
@@ -343,56 +400,72 @@ impl Ignore {
path: &Path,
is_dir: bool,
) -> Match<IgnoreMatch<'a>> {
let (mut m_custom_ignore, mut m_ignore, mut m_gi, mut m_gi_exclude, mut m_explicit) =
(Match::None, Match::None, Match::None, Match::None, Match::None);
let any_git = self.parents().any(|ig| ig.0.has_git);
let (
mut m_custom_ignore,
mut m_ignore,
mut m_gi,
mut m_gi_exclude,
mut m_explicit,
) = (Match::None, Match::None, Match::None, Match::None, Match::None);
let any_git =
!self.0.opts.require_git || self.parents().any(|ig| ig.0.has_git);
let mut saw_git = false;
for ig in self.parents().take_while(|ig| !ig.0.is_absolute_parent) {
if m_custom_ignore.is_none() {
m_custom_ignore =
ig.0.custom_ignore_matcher.matched(path, is_dir)
.map(IgnoreMatch::gitignore);
ig.0.custom_ignore_matcher
.matched(path, is_dir)
.map(IgnoreMatch::gitignore);
}
if m_ignore.is_none() {
m_ignore =
ig.0.ignore_matcher.matched(path, is_dir)
.map(IgnoreMatch::gitignore);
ig.0.ignore_matcher
.matched(path, is_dir)
.map(IgnoreMatch::gitignore);
}
if any_git && !saw_git && m_gi.is_none() {
m_gi =
ig.0.git_ignore_matcher.matched(path, is_dir)
.map(IgnoreMatch::gitignore);
ig.0.git_ignore_matcher
.matched(path, is_dir)
.map(IgnoreMatch::gitignore);
}
if any_git && !saw_git && m_gi_exclude.is_none() {
m_gi_exclude =
ig.0.git_exclude_matcher.matched(path, is_dir)
.map(IgnoreMatch::gitignore);
ig.0.git_exclude_matcher
.matched(path, is_dir)
.map(IgnoreMatch::gitignore);
}
saw_git = saw_git || ig.0.has_git;
}
if self.0.opts.parents {
if let Some(abs_parent_path) = self.absolute_base() {
let path = abs_parent_path.join(path);
for ig in self.parents().skip_while(|ig|!ig.0.is_absolute_parent) {
for ig in
self.parents().skip_while(|ig| !ig.0.is_absolute_parent)
{
if m_custom_ignore.is_none() {
m_custom_ignore =
ig.0.custom_ignore_matcher.matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
ig.0.custom_ignore_matcher
.matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
}
if m_ignore.is_none() {
m_ignore =
ig.0.ignore_matcher.matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
ig.0.ignore_matcher
.matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
}
if any_git && !saw_git && m_gi.is_none() {
m_gi =
ig.0.git_ignore_matcher.matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
ig.0.git_ignore_matcher
.matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
}
if any_git && !saw_git && m_gi_exclude.is_none() {
m_gi_exclude =
ig.0.git_exclude_matcher.matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
ig.0.git_exclude_matcher
.matched(&path, is_dir)
.map(IgnoreMatch::gitignore);
}
saw_git = saw_git || ig.0.has_git;
}
@@ -404,16 +477,21 @@ impl Ignore {
}
m_explicit = gi.matched(&path, is_dir).map(IgnoreMatch::gitignore);
}
let m_global =
if any_git {
self.0.git_global_matcher
.matched(&path, is_dir)
.map(IgnoreMatch::gitignore)
} else {
Match::None
};
let m_global = if any_git {
self.0
.git_global_matcher
.matched(&path, is_dir)
.map(IgnoreMatch::gitignore)
} else {
Match::None
};
m_custom_ignore.or(m_ignore).or(m_gi).or(m_gi_exclude).or(m_global).or(m_explicit)
m_custom_ignore
.or(m_ignore)
.or(m_gi)
.or(m_gi_exclude)
.or(m_global)
.or(m_explicit)
}
/// Returns an iterator over parent ignore matchers, including this one.
@@ -483,6 +561,8 @@ impl IgnoreBuilder {
git_global: true,
git_ignore: true,
git_exclude: true,
ignore_case_insensitive: false,
require_git: true,
},
}
}
@@ -492,16 +572,19 @@ impl IgnoreBuilder {
/// The matcher returned won't match anything until ignore rules from
/// directories are added to it.
pub fn build(&self) -> Ignore {
let git_global_matcher =
if !self.opts.git_global {
Gitignore::empty()
} else {
let (gi, err) = Gitignore::global();
if let Some(err) = err {
debug!("{}", err);
}
gi
};
let git_global_matcher = if !self.opts.git_global {
Gitignore::empty()
} else {
let mut builder = GitignoreBuilder::new("");
builder
.case_insensitive(self.opts.ignore_case_insensitive)
.unwrap();
let (gi, err) = builder.build_global();
if let Some(err) = err {
debug!("{}", err);
}
gi
};
Ignore(Arc::new(IgnoreInner {
compiled: Arc::new(RwLock::new(HashMap::new())),
@@ -512,7 +595,9 @@ impl IgnoreBuilder {
is_absolute_parent: true,
absolute_base: None,
explicit_ignores: Arc::new(self.explicit_ignores.clone()),
custom_ignore_filenames: Arc::new(self.custom_ignore_filenames.clone()),
custom_ignore_filenames: Arc::new(
self.custom_ignore_filenames.clone(),
),
custom_ignore_matcher: Gitignore::empty(),
ignore_matcher: Gitignore::empty(),
git_global_matcher: Arc::new(git_global_matcher),
@@ -557,7 +642,7 @@ impl IgnoreBuilder {
/// later names.
pub fn add_custom_ignore_filename<S: AsRef<OsStr>>(
&mut self,
file_name: S
file_name: S,
) -> &mut IgnoreBuilder {
self.custom_ignore_filenames.push(file_name.as_ref().to_os_string());
self
@@ -627,23 +712,63 @@ impl IgnoreBuilder {
self.opts.git_exclude = yes;
self
}
/// Whether a git repository is required to apply git-related ignore
/// rules (global rules, .gitignore and local exclude rules).
///
/// When disabled, git-related ignore rules are applied even when searching
/// outside a git repository.
pub fn require_git(&mut self, yes: bool) -> &mut IgnoreBuilder {
self.opts.require_git = yes;
self
}
/// Process ignore files case insensitively
///
/// This is disabled by default.
pub fn ignore_case_insensitive(
&mut self,
yes: bool,
) -> &mut IgnoreBuilder {
self.opts.ignore_case_insensitive = yes;
self
}
}
/// Creates a new gitignore matcher for the directory given.
///
/// Ignore globs are extracted from each of the file names in `dir` in the
/// order given (earlier names have lower precedence than later names).
/// The matcher is meant to match files below `dir`.
/// Ignore globs are extracted from each of the file names relative to
/// `dir_for_ignorefile` in the order given (earlier names have lower
/// precedence than later names).
///
/// I/O errors are ignored.
pub fn create_gitignore<T: AsRef<OsStr>>(
dir: &Path,
dir_for_ignorefile: &Path,
names: &[T],
case_insensitive: bool,
) -> (Gitignore, Option<Error>) {
let mut builder = GitignoreBuilder::new(dir);
let mut errs = PartialErrorBuilder::default();
builder.case_insensitive(case_insensitive).unwrap();
for name in names {
let gipath = dir.join(name.as_ref());
errs.maybe_push_ignore_io(builder.add(gipath));
let gipath = dir_for_ignorefile.join(name.as_ref());
// This check is not necessary, but is added for performance. Namely,
// a simple stat call checking for existence can often be just a bit
// quicker than actually trying to open a file. Since the number of
// directories without ignore files likely greatly exceeds the number
// with ignore files, this check generally makes sense.
//
// However, until demonstrated otherwise, we speculatively do not do
// this on Windows since Windows is notorious for having slow file
// system operations. Namely, it's not clear whether this analysis
// makes sense on Windows.
//
// For more details: https://github.com/BurntSushi/ripgrep/pull/1381
if cfg!(windows) || gipath.exists() {
errs.maybe_push_ignore_io(builder.add(gipath));
}
}
let gi = match builder.build() {
Ok(gi) => gi,
@@ -655,16 +780,69 @@ pub fn create_gitignore<T: AsRef<OsStr>>(
(gi, errs.into_error_option())
}
/// Find the GIT_COMMON_DIR for the given git worktree.
///
/// This is the directory that may contain a private ignore file
/// "info/exclude". Unlike git, this function does *not* read environment
/// variables GIT_DIR and GIT_COMMON_DIR, because it is not clear how to use
/// them when multiple repositories are searched.
///
/// Some I/O errors are ignored.
fn resolve_git_commondir(
dir: &Path,
git_type: Option<FileType>,
) -> Result<PathBuf, Option<Error>> {
let git_dir_path = || dir.join(".git");
let git_dir = git_dir_path();
if !git_type.map_or(false, |ft| ft.is_file()) {
return Ok(git_dir);
}
let file = match File::open(git_dir) {
Ok(file) => io::BufReader::new(file),
Err(err) => {
return Err(Some(Error::Io(err).with_path(git_dir_path())));
}
};
let dot_git_line = match file.lines().next() {
Some(Ok(line)) => line,
Some(Err(err)) => {
return Err(Some(Error::Io(err).with_path(git_dir_path())));
}
None => return Err(None),
};
if !dot_git_line.starts_with("gitdir: ") {
return Err(None);
}
let real_git_dir = PathBuf::from(&dot_git_line["gitdir: ".len()..]);
let git_commondir_file = || real_git_dir.join("commondir");
let file = match File::open(git_commondir_file()) {
Ok(file) => io::BufReader::new(file),
Err(_) => return Err(None),
};
let commondir_line = match file.lines().next() {
Some(Ok(line)) => line,
Some(Err(err)) => {
return Err(Some(Error::Io(err).with_path(git_commondir_file())));
}
None => return Err(None),
};
let commondir_abs = if commondir_line.starts_with(".") {
real_git_dir.join(commondir_line) // relative commondir
} else {
PathBuf::from(commondir_line)
};
Ok(commondir_abs)
}
#[cfg(test)]
mod tests {
use std::fs::{self, File};
use std::io::Write;
use std::path::Path;
use tempdir::TempDir;
use dir::IgnoreBuilder;
use gitignore::Gitignore;
use tests::TempDir;
use Error;
fn wfile<P: AsRef<Path>>(path: P, contents: &str) {
@@ -683,15 +861,19 @@ mod tests {
}
}
fn tmpdir() -> TempDir {
TempDir::new().unwrap()
}
#[test]
fn explicit_ignore() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
wfile(td.path().join("not-an-ignore"), "foo\n!bar");
let (gi, err) = Gitignore::new(td.path().join("not-an-ignore"));
assert!(err.is_none());
let (ig, err) = IgnoreBuilder::new()
.add_ignore(gi).build().add_child(td.path());
let (ig, err) =
IgnoreBuilder::new().add_ignore(gi).build().add_child(td.path());
assert!(err.is_none());
assert!(ig.matched("foo", false).is_ignore());
assert!(ig.matched("bar", false).is_whitelist());
@@ -700,7 +882,7 @@ mod tests {
#[test]
fn git_exclude() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
mkdirp(td.path().join(".git/info"));
wfile(td.path().join(".git/info/exclude"), "foo\n!bar");
@@ -713,7 +895,7 @@ mod tests {
#[test]
fn gitignore() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
mkdirp(td.path().join(".git"));
wfile(td.path().join(".gitignore"), "foo\n!bar");
@@ -726,7 +908,7 @@ mod tests {
#[test]
fn gitignore_no_git() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
wfile(td.path().join(".gitignore"), "foo\n!bar");
let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());
@@ -736,9 +918,24 @@ mod tests {
assert!(ig.matched("baz", false).is_none());
}
#[test]
fn gitignore_allowed_no_git() {
let td = tmpdir();
wfile(td.path().join(".gitignore"), "foo\n!bar");
let (ig, err) = IgnoreBuilder::new()
.require_git(false)
.build()
.add_child(td.path());
assert!(err.is_none());
assert!(ig.matched("foo", false).is_ignore());
assert!(ig.matched("bar", false).is_whitelist());
assert!(ig.matched("baz", false).is_none());
}
#[test]
fn ignore() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
wfile(td.path().join(".ignore"), "foo\n!bar");
let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());
@@ -750,13 +947,14 @@ mod tests {
#[test]
fn custom_ignore() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
let custom_ignore = ".customignore";
wfile(td.path().join(custom_ignore), "foo\n!bar");
let (ig, err) = IgnoreBuilder::new()
.add_custom_ignore_filename(custom_ignore)
.build().add_child(td.path());
.build()
.add_child(td.path());
assert!(err.is_none());
assert!(ig.matched("foo", false).is_ignore());
assert!(ig.matched("bar", false).is_whitelist());
@@ -766,14 +964,15 @@ mod tests {
// Tests that a custom ignore file will override an .ignore.
#[test]
fn custom_ignore_over_ignore() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
let custom_ignore = ".customignore";
wfile(td.path().join(".ignore"), "foo");
wfile(td.path().join(custom_ignore), "!foo");
let (ig, err) = IgnoreBuilder::new()
.add_custom_ignore_filename(custom_ignore)
.build().add_child(td.path());
.build()
.add_child(td.path());
assert!(err.is_none());
assert!(ig.matched("foo", false).is_whitelist());
}
@@ -781,7 +980,7 @@ mod tests {
// Tests that earlier custom ignore files have lower precedence than later.
#[test]
fn custom_ignore_precedence() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
let custom_ignore1 = ".customignore1";
let custom_ignore2 = ".customignore2";
wfile(td.path().join(custom_ignore1), "foo");
@@ -790,7 +989,8 @@ mod tests {
let (ig, err) = IgnoreBuilder::new()
.add_custom_ignore_filename(custom_ignore1)
.add_custom_ignore_filename(custom_ignore2)
.build().add_child(td.path());
.build()
.add_child(td.path());
assert!(err.is_none());
assert!(ig.matched("foo", false).is_whitelist());
}
@@ -798,7 +998,7 @@ mod tests {
// Tests that an .ignore will override a .gitignore.
#[test]
fn ignore_over_gitignore() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
wfile(td.path().join(".gitignore"), "foo");
wfile(td.path().join(".ignore"), "!foo");
@@ -810,7 +1010,7 @@ mod tests {
// Tests that exclude has lower precedent than both .ignore and .gitignore.
#[test]
fn exclude_lowest() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
wfile(td.path().join(".gitignore"), "!foo");
wfile(td.path().join(".ignore"), "!bar");
mkdirp(td.path().join(".git/info"));
@@ -825,8 +1025,8 @@ mod tests {
#[test]
fn errored() {
let td = TempDir::new("ignore-test-").unwrap();
wfile(td.path().join(".gitignore"), "f**oo");
let td = tmpdir();
wfile(td.path().join(".gitignore"), "{foo");
let (_, err) = IgnoreBuilder::new().build().add_child(td.path());
assert!(err.is_some());
@@ -834,9 +1034,9 @@ mod tests {
#[test]
fn errored_both() {
let td = TempDir::new("ignore-test-").unwrap();
wfile(td.path().join(".gitignore"), "f**oo");
wfile(td.path().join(".ignore"), "fo**o");
let td = tmpdir();
wfile(td.path().join(".gitignore"), "{foo");
wfile(td.path().join(".ignore"), "{bar");
let (_, err) = IgnoreBuilder::new().build().add_child(td.path());
assert_eq!(2, partial(err.expect("an error")).len());
@@ -844,9 +1044,9 @@ mod tests {
#[test]
fn errored_partial() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
mkdirp(td.path().join(".git"));
wfile(td.path().join(".gitignore"), "f**oo\nbar");
wfile(td.path().join(".gitignore"), "{foo\nbar");
let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());
assert!(err.is_some());
@@ -855,8 +1055,8 @@ mod tests {
#[test]
fn errored_partial_and_ignore() {
let td = TempDir::new("ignore-test-").unwrap();
wfile(td.path().join(".gitignore"), "f**oo\nbar");
let td = tmpdir();
wfile(td.path().join(".gitignore"), "{foo\nbar");
wfile(td.path().join(".ignore"), "!bar");
let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());
@@ -866,7 +1066,7 @@ mod tests {
#[test]
fn not_present_empty() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
let (_, err) = IgnoreBuilder::new().build().add_child(td.path());
assert!(err.is_none());
@@ -876,7 +1076,7 @@ mod tests {
fn stops_at_git_dir() {
// This tests that .gitignore files beyond a .git barrier aren't
// matched, but .ignore files are.
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
mkdirp(td.path().join(".git"));
mkdirp(td.path().join("foo/.git"));
wfile(td.path().join(".gitignore"), "foo");
@@ -897,7 +1097,7 @@ mod tests {
#[test]
fn absolute_parent() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
mkdirp(td.path().join(".git"));
mkdirp(td.path().join("foo"));
wfile(td.path().join(".gitignore"), "bar");
@@ -920,7 +1120,7 @@ mod tests {
#[test]
fn absolute_parent_anchored() {
let td = TempDir::new("ignore-test-").unwrap();
let td = tmpdir();
mkdirp(td.path().join(".git"));
mkdirp(td.path().join("src/llvm"));
wfile(td.path().join(".gitignore"), "/llvm/\nfoo");
@@ -937,4 +1137,49 @@ mod tests {
assert!(ig2.matched("foo", false).is_ignore());
assert!(ig2.matched("src/foo", false).is_ignore());
}
#[test]
fn git_info_exclude_in_linked_worktree() {
let td = tmpdir();
let git_dir = td.path().join(".git");
mkdirp(git_dir.join("info"));
wfile(git_dir.join("info/exclude"), "ignore_me");
mkdirp(git_dir.join("worktrees/linked-worktree"));
let commondir_path =
|| git_dir.join("worktrees/linked-worktree/commondir");
mkdirp(td.path().join("linked-worktree"));
let worktree_git_dir_abs = format!(
"gitdir: {}",
git_dir.join("worktrees/linked-worktree").to_str().unwrap(),
);
wfile(td.path().join("linked-worktree/.git"), &worktree_git_dir_abs);
// relative commondir
wfile(commondir_path(), "../..");
let ib = IgnoreBuilder::new().build();
let (ignore, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_none());
assert!(ignore.matched("ignore_me", false).is_ignore());
// absolute commondir
wfile(commondir_path(), git_dir.to_str().unwrap());
let (ignore, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_none());
assert!(ignore.matched("ignore_me", false).is_ignore());
// missing commondir file
assert!(fs::remove_file(commondir_path()).is_ok());
let (_, err) = ib.add_child(td.path().join("linked-worktree"));
// We squash the error in this case, because it occurs in repositories
// that are not linked worktrees but have submodules.
assert!(err.is_none());
wfile(td.path().join("linked-worktree/.git"), "garbage");
let (_, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_none());
wfile(td.path().join("linked-worktree/.git"), "gitdir: garbage");
let (_, err) = ib.add_child(td.path().join("linked-worktree"));
assert!(err.is_none());
}
}

134
ignore/src/gitignore.rs → crates/ignore/src/gitignore.rs
View File

@@ -69,8 +69,7 @@ impl Glob {
/// Returns true if and only if this glob has a `**/` prefix.
fn has_doublestar_prefix(&self) -> bool {
self.actual.starts_with("**/")
|| (self.actual == "**" && self.is_only_dir)
self.actual.starts_with("**/") || self.actual == "**"
}
}
@@ -127,16 +126,7 @@ impl Gitignore {
/// `$XDG_CONFIG_HOME/git/ignore` is read. If `$XDG_CONFIG_HOME` is not
/// set or is empty, then `$HOME/.config/git/ignore` is used instead.
pub fn global() -> (Gitignore, Option<Error>) {
match gitconfig_excludes_path() {
None => (Gitignore::empty(), None),
Some(path) => {
if !path.is_file() {
(Gitignore::empty(), None)
} else {
Gitignore::new(path)
}
}
}
GitignoreBuilder::new("").build_global()
}
/// Creates a new empty gitignore matcher that never matches anything.
@@ -259,7 +249,7 @@ impl Gitignore {
return Match::None;
}
let path = path.as_ref();
let _matches = self.matches.as_ref().unwrap().get_default();
let _matches = self.matches.as_ref().unwrap().get_or_default();
let mut matches = _matches.borrow_mut();
let candidate = Candidate::new(path);
self.set.matches_candidate_into(&candidate, &mut *matches);
@@ -342,13 +332,10 @@ impl GitignoreBuilder {
pub fn build(&self) -> Result<Gitignore, Error> {
let nignore = self.globs.iter().filter(|g| !g.is_whitelist()).count();
let nwhite = self.globs.iter().filter(|g| g.is_whitelist()).count();
let set =
self.builder.build().map_err(|err| {
Error::Glob {
glob: None,
err: err.to_string(),
}
})?;
let set = self
.builder
.build()
.map_err(|err| Error::Glob { glob: None, err: err.to_string() })?;
Ok(Gitignore {
set: set,
root: self.root.clone(),
@@ -359,6 +346,36 @@ impl GitignoreBuilder {
})
}
/// Build a global gitignore matcher using the configuration in this
/// builder.
///
/// This consumes ownership of the builder unlike `build` because it
/// must mutate the builder to add the global gitignore globs.
///
/// Note that this ignores the path given to this builder's constructor
/// and instead derives the path automatically from git's global
/// configuration.
pub fn build_global(mut self) -> (Gitignore, Option<Error>) {
match gitconfig_excludes_path() {
None => (Gitignore::empty(), None),
Some(path) => {
if !path.is_file() {
(Gitignore::empty(), None)
} else {
let mut errs = PartialErrorBuilder::default();
errs.maybe_push_ignore_io(self.add(path));
match self.build() {
Ok(gi) => (gi, errs.into_error_option()),
Err(err) => {
errs.push(err);
(Gitignore::empty(), errs.into_error_option())
}
}
}
}
}
}
/// Add each glob from the file path given.
///
/// The file given should be formatted as a `gitignore` file.
@@ -419,6 +436,8 @@ impl GitignoreBuilder {
from: Option<PathBuf>,
mut line: &str,
) -> Result<&mut GitignoreBuilder, Error> {
#![allow(deprecated)]
if line.starts_with("#") {
return Ok(self);
}
@@ -435,7 +454,6 @@ impl GitignoreBuilder {
is_whitelist: false,
is_only_dir: false,
};
let mut literal_separator = false;
let mut is_absolute = false;
if line.starts_with("\\!") || line.starts_with("\\#") {
line = &line[1..];
@@ -450,7 +468,6 @@ impl GitignoreBuilder {
// then the glob can only match the beginning of a path
// (relative to the location of gitignore). We achieve this by
// simply banning wildcards from matching /.
literal_separator = true;
line = &line[1..];
is_absolute = true;
}
@@ -463,16 +480,11 @@ impl GitignoreBuilder {
line = &line[..i];
}
}
// If there is a literal slash, then we note that so that globbing
// doesn't let wildcards match slashes.
glob.actual = line.to_string();
if is_absolute || line.chars().any(|c| c == '/') {
literal_separator = true;
}
// If there was a slash, then this is a glob that must match the entire
// path name. Otherwise, we should let it match anywhere, so use a **/
// prefix.
if !literal_separator {
// If there is a literal slash, then this is a glob that must match the
// entire path name. Otherwise, we should let it match anywhere, so use
// a **/ prefix.
if !is_absolute && !line.chars().any(|c| c == '/') {
// ... but only if we don't already have a **/ prefix.
if !glob.has_doublestar_prefix() {
glob.actual = format!("**/{}", glob.actual);
@@ -484,18 +496,15 @@ impl GitignoreBuilder {
if glob.actual.ends_with("/**") {
glob.actual = format!("{}/*", glob.actual);
}
let parsed =
GlobBuilder::new(&glob.actual)
.literal_separator(literal_separator)
.case_insensitive(self.case_insensitive)
.backslash_escape(true)
.build()
.map_err(|err| {
Error::Glob {
glob: Some(glob.original.clone()),
err: err.kind().to_string(),
}
})?;
let parsed = GlobBuilder::new(&glob.actual)
.literal_separator(true)
.case_insensitive(self.case_insensitive)
.backslash_escape(true)
.build()
.map_err(|err| Error::Glob {
glob: Some(glob.original.clone()),
err: err.kind().to_string(),
})?;
self.builder.add(parsed);
self.globs.push(glob);
Ok(self)
@@ -503,12 +512,16 @@ impl GitignoreBuilder {
/// Toggle whether the globs should be matched case insensitively or not.
///
/// When this option is changed, only globs added after the change will be affected.
/// When this option is changed, only globs added after the change will be
/// affected.
///
/// This is disabled by default.
pub fn case_insensitive(
&mut self, yes: bool
&mut self,
yes: bool,
) -> Result<&mut GitignoreBuilder, Error> {
// TODO: This should not return a `Result`. Fix this in the next semver
// release.
self.case_insensitive = yes;
Ok(self)
}
@@ -518,7 +531,7 @@ impl GitignoreBuilder {
///
/// Note that the file path returned may not exist.
fn gitconfig_excludes_path() -> Option<PathBuf> {
// git supports $HOME/.gitconfig and $XDG_CONFIG_DIR/git/config. Notably,
// git supports $HOME/.gitconfig and $XDG_CONFIG_HOME/git/config. Notably,
// both can be active at the same time, where $HOME/.gitconfig takes
// precedent. So if $HOME/.gitconfig defines a `core.excludesFile`, then
// we're done.
@@ -549,7 +562,7 @@ fn gitconfig_home_contents() -> Option<Vec<u8>> {
}
/// Returns the file contents of git's global config file, if one exists, in
/// the user's XDG_CONFIG_DIR directory.
/// the user's XDG_CONFIG_HOME directory.
fn gitconfig_xdg_contents() -> Option<Vec<u8>> {
let path = env::var_os("XDG_CONFIG_HOME")
.and_then(|x| if x.is_empty() { None } else { Some(PathBuf::from(x)) })
@@ -580,9 +593,8 @@ fn parse_excludes_file(data: &[u8]) -> Option<PathBuf> {
// probably works in more circumstances. I guess we would ideally have
// a full INI parser. Yuck.
lazy_static! {
static ref RE: Regex = Regex::new(
r"(?im)^\s*excludesfile\s*=\s*(.+)\s*$"
).unwrap();
static ref RE: Regex =
Regex::new(r"(?im)^\s*excludesfile\s*=\s*(.+)\s*$").unwrap();
};
let caps = match RE.captures(data) {
None => return None,
@@ -611,8 +623,8 @@ fn home_dir() -> Option<PathBuf> {
#[cfg(test)]
mod tests {
use std::path::Path;
use super::{Gitignore, GitignoreBuilder};
use std::path::Path;
fn gi_from_str<P: AsRef<Path>>(root: P, s: &str) -> Gitignore {
let mut builder = GitignoreBuilder::new(root);
@@ -689,6 +701,9 @@ mod tests {
ignored!(ig39, ROOT, "\\?", "?");
ignored!(ig40, ROOT, "\\*", "*");
ignored!(ig41, ROOT, "\\a", "a");
ignored!(ig42, ROOT, "s*.rs", "sfoo.rs");
ignored!(ig43, ROOT, "**", "foo.rs");
ignored!(ig44, ROOT, "**/**/*", "a/foo.rs");
not_ignored!(ignot1, ROOT, "amonths", "months");
not_ignored!(ignot2, ROOT, "monthsa", "months");
@@ -704,12 +719,16 @@ mod tests {
not_ignored!(ignot12, ROOT, "\n\n\n", "foo");
not_ignored!(ignot13, ROOT, "foo/**", "foo", true);
not_ignored!(
ignot14, "./third_party/protobuf", "m4/ltoptions.m4",
"./third_party/protobuf/csharp/src/packages/repositories.config");
ignot14,
"./third_party/protobuf",
"m4/ltoptions.m4",
"./third_party/protobuf/csharp/src/packages/repositories.config"
);
not_ignored!(ignot15, ROOT, "!/bar", "foo/bar");
not_ignored!(ignot16, ROOT, "*\n!**/", "foo", true);
not_ignored!(ignot17, ROOT, "src/*.rs", "src/grep/src/main.rs");
not_ignored!(ignot18, ROOT, "path1/*", "path2/path1/foo");
not_ignored!(ignot19, ROOT, "s*.rs", "src/foo.rs");
fn bytes(s: &str) -> Vec<u8> {
s.to_string().into_bytes()
@@ -748,9 +767,12 @@ mod tests {
#[test]
fn case_insensitive() {
let gi = GitignoreBuilder::new(ROOT)
.case_insensitive(true).unwrap()
.add_str(None, "*.html").unwrap()
.build().unwrap();
.case_insensitive(true)
.unwrap()
.add_str(None, "*.html")
.unwrap()
.build()
.unwrap();
assert!(gi.matched("foo.html", false).is_ignore());
assert!(gi.matched("foo.HTML", false).is_ignore());
assert!(!gi.matched("foo.htm", false).is_ignore());

132
ignore/src/lib.rs → crates/ignore/src/lib.rs
View File

@@ -46,7 +46,6 @@ See the documentation for `WalkBuilder` for many other options.
#![deny(missing_docs)]
extern crate crossbeam_channel as channel;
extern crate globset;
#[macro_use]
extern crate lazy_static;
@@ -55,8 +54,6 @@ extern crate log;
extern crate memchr;
extern crate regex;
extern crate same_file;
#[cfg(test)]
extern crate tempdir;
extern crate thread_local;
extern crate walkdir;
#[cfg(windows)]
@@ -67,12 +64,16 @@ use std::fmt;
use std::io;
use std::path::{Path, PathBuf};
pub use walk::{DirEntry, Walk, WalkBuilder, WalkParallel, WalkState};
pub use walk::{
DirEntry, ParallelVisitor, ParallelVisitorBuilder, Walk, WalkBuilder,
WalkParallel, WalkState,
};
mod default_types;
mod dir;
pub mod gitignore;
mod pathutil;
pub mod overrides;
mod pathutil;
pub mod types;
mod walk;
@@ -145,20 +146,14 @@ impl Clone for Error {
Error::WithDepth { depth, ref err } => {
Error::WithDepth { depth: depth, err: err.clone() }
}
Error::Loop { ref ancestor, ref child } => {
Error::Loop {
ancestor: ancestor.clone(),
child: child.clone()
}
}
Error::Io(ref err) => {
match err.raw_os_error() {
Some(e) => Error::Io(io::Error::from_raw_os_error(e)),
None => {
Error::Io(io::Error::new(err.kind(), err.to_string()))
}
}
}
Error::Loop { ref ancestor, ref child } => Error::Loop {
ancestor: ancestor.clone(),
child: child.clone(),
},
Error::Io(ref err) => match err.raw_os_error() {
Some(e) => Error::Io(io::Error::from_raw_os_error(e)),
None => Error::Io(io::Error::new(err.kind(), err.to_string())),
},
Error::Glob { ref glob, ref err } => {
Error::Glob { glob: glob.clone(), err: err.clone() }
}
@@ -221,19 +216,14 @@ impl Error {
/// Turn an error into a tagged error with the given depth.
fn with_depth(self, depth: usize) -> Error {
Error::WithDepth {
depth: depth,
err: Box::new(self),
}
Error::WithDepth { depth: depth, err: Box::new(self) }
}
/// Turn an error into a tagged error with the given file path and line
/// number. If path is empty, then it is omitted from the error.
fn tagged<P: AsRef<Path>>(self, path: P, lineno: u64) -> Error {
let errline = Error::WithLineNumber {
line: lineno,
err: Box::new(self),
};
let errline =
Error::WithLineNumber { line: lineno, err: Box::new(self) };
if path.as_ref().as_os_str().is_empty() {
return errline;
}
@@ -255,16 +245,14 @@ impl Error {
let path = err.path().map(|p| p.to_path_buf());
let mut ig_err = Error::Io(io::Error::from(err));
if let Some(path) = path {
ig_err = Error::WithPath {
path: path,
err: Box::new(ig_err),
};
ig_err = Error::WithPath { path: path, err: Box::new(ig_err) };
}
ig_err
}
}
impl error::Error for Error {
#[allow(deprecated)]
fn description(&self) -> &str {
match *self {
Error::Partial(_) => "partial error",
@@ -295,11 +283,13 @@ impl fmt::Display for Error {
write!(f, "{}: {}", path.display(), err)
}
Error::WithDepth { ref err, .. } => err.fmt(f),
Error::Loop { ref ancestor, ref child } => {
write!(f, "File system loop found: \
Error::Loop { ref ancestor, ref child } => write!(
f,
"File system loop found: \
{} points to an ancestor {}",
child.display(), ancestor.display())
}
child.display(),
ancestor.display()
),
Error::Io(ref err) => err.fmt(f),
Error::Glob { glob: None, ref err } => write!(f, "{}", err),
Error::Glob { glob: Some(ref glob), ref err } => {
@@ -308,10 +298,11 @@ impl fmt::Display for Error {
Error::UnrecognizedFileType(ref ty) => {
write!(f, "unrecognized file type: {}", ty)
}
Error::InvalidDefinition => {
write!(f, "invalid definition (format is type:glob, e.g., \
html:*.html)")
}
Error::InvalidDefinition => write!(
f,
"invalid definition (format is type:glob, e.g., \
html:*.html)"
),
}
}
}
@@ -442,3 +433,66 @@ impl<T> Match<T> {
}
}
}
#[cfg(test)]
mod tests {
use std::env;
use std::error;
use std::fs;
use std::path::{Path, PathBuf};
use std::result;
/// A convenient result type alias.
pub type Result<T> =
result::Result<T, Box<dyn error::Error + Send + Sync>>;
macro_rules! err {
($($tt:tt)*) => {
Box::<dyn error::Error + Send + Sync>::from(format!($($tt)*))
}
}
/// A simple wrapper for creating a temporary directory that is
/// automatically deleted when it's dropped.
///
/// We use this in lieu of tempfile because tempfile brings in too many
/// dependencies.
#[derive(Debug)]
pub struct TempDir(PathBuf);
impl Drop for TempDir {
fn drop(&mut self) {
fs::remove_dir_all(&self.0).unwrap();
}
}
impl TempDir {
/// Create a new empty temporary directory under the system's configured
/// temporary directory.
pub fn new() -> Result<TempDir> {
use std::sync::atomic::{AtomicUsize, Ordering};
static TRIES: usize = 100;
static COUNTER: AtomicUsize = AtomicUsize::new(0);
let tmpdir = env::temp_dir();
for _ in 0..TRIES {
let count = COUNTER.fetch_add(1, Ordering::SeqCst);
let path = tmpdir.join("rust-ignore").join(count.to_string());
if path.is_dir() {
continue;
}
fs::create_dir_all(&path).map_err(|e| {
err!("failed to create {}: {}", path.display(), e)
})?;
return Ok(TempDir(path));
}
Err(err!("failed to create temp dir after {} tries", TRIES))
}
/// Return the underlying path to this temporary directory.
pub fn path(&self) -> &Path {
&self.0
}
}
}

25
ignore/src/overrides.rs → crates/ignore/src/overrides.rs
View File

@@ -115,9 +115,7 @@ impl OverrideBuilder {
///
/// Matching is done relative to the directory path provided.
pub fn new<P: AsRef<Path>>(path: P) -> OverrideBuilder {
OverrideBuilder {
builder: GitignoreBuilder::new(path),
}
OverrideBuilder { builder: GitignoreBuilder::new(path) }
}
/// Builds a new override matcher from the globs added so far.
@@ -139,13 +137,16 @@ impl OverrideBuilder {
}
/// Toggle whether the globs should be matched case insensitively or not.
///
///
/// When this option is changed, only globs added after the change will be affected.
///
/// This is disabled by default.
pub fn case_insensitive(
&mut self, yes: bool
&mut self,
yes: bool,
) -> Result<&mut OverrideBuilder, Error> {
// TODO: This should not return a `Result`. Fix this in the next semver
// release.
self.builder.case_insensitive(yes)?;
Ok(self)
}
@@ -237,9 +238,12 @@ mod tests {
#[test]
fn case_insensitive() {
let ov = OverrideBuilder::new(ROOT)
.case_insensitive(true).unwrap()
.add("*.html").unwrap()
.build().unwrap();
.case_insensitive(true)
.unwrap()
.add("*.html")
.unwrap()
.build()
.unwrap();
assert!(ov.matched("foo.html", false).is_whitelist());
assert!(ov.matched("foo.HTML", false).is_whitelist());
assert!(ov.matched("foo.htm", false).is_ignore());
@@ -248,9 +252,8 @@ mod tests {
#[test]
fn default_case_sensitive() {
let ov = OverrideBuilder::new(ROOT)
.add("*.html").unwrap()
.build().unwrap();
let ov =
OverrideBuilder::new(ROOT).add("*.html").unwrap().build().unwrap();
assert!(ov.matched("foo.html", false).is_whitelist());
assert!(ov.matched("foo.HTML", false).is_ignore());
assert!(ov.matched("foo.htm", false).is_ignore());

52
ignore/src/pathutil.rs → crates/ignore/src/pathutil.rs
View File

@@ -1,22 +1,56 @@
use std::ffi::OsStr;
use std::path::Path;
/// Returns true if and only if this file path is considered to be hidden.
use walk::DirEntry;
/// Returns true if and only if this entry is considered to be hidden.
///
/// This only returns true if the base name of the path starts with a `.`.
///
/// On Unix, this implements a more optimized check.
#[cfg(unix)]
pub fn is_hidden<P: AsRef<Path>>(path: P) -> bool {
pub fn is_hidden(dent: &DirEntry) -> bool {
use std::os::unix::ffi::OsStrExt;
if let Some(name) = file_name(path.as_ref()) {
if let Some(name) = file_name(dent.path()) {
name.as_bytes().get(0) == Some(&b'.')
} else {
false
}
}
/// Returns true if and only if this file path is considered to be hidden.
#[cfg(not(unix))]
pub fn is_hidden<P: AsRef<Path>>(path: P) -> bool {
if let Some(name) = file_name(path.as_ref()) {
/// Returns true if and only if this entry is considered to be hidden.
///
/// On Windows, this returns true if one of the following is true:
///
/// * The base name of the path starts with a `.`.
/// * The file attributes have the `HIDDEN` property set.
#[cfg(windows)]
pub fn is_hidden(dent: &DirEntry) -> bool {
use std::os::windows::fs::MetadataExt;
use winapi_util::file;
// This looks like we're doing an extra stat call, but on Windows, the
// directory traverser reuses the metadata retrieved from each directory
// entry and stores it on the DirEntry itself. So this is "free."
if let Ok(md) = dent.metadata() {
if file::is_hidden(md.file_attributes() as u64) {
return true;
}
}
if let Some(name) = file_name(dent.path()) {
name.to_str().map(|s| s.starts_with(".")).unwrap_or(false)
} else {
false
}
}
/// Returns true if and only if this entry is considered to be hidden.
///
/// This only returns true if the base name of the path starts with a `.`.
#[cfg(not(any(unix, windows)))]
pub fn is_hidden(dent: &DirEntry) -> bool {
if let Some(name) = file_name(dent.path()) {
name.to_str().map(|s| s.starts_with(".")).unwrap_or(false)
} else {
false
@@ -57,8 +91,8 @@ pub fn strip_prefix<'a, P: AsRef<Path> + ?Sized>(
/// the empty string.
#[cfg(unix)]
pub fn is_file_name<P: AsRef<Path>>(path: P) -> bool {
use std::os::unix::ffi::OsStrExt;
use memchr::memchr;
use std::os::unix::ffi::OsStrExt;
let path = path.as_ref().as_os_str().as_bytes();
memchr(b'/', path).is_none()
@@ -79,8 +113,8 @@ pub fn is_file_name<P: AsRef<Path>>(path: P) -> bool {
pub fn file_name<'a, P: AsRef<Path> + ?Sized>(
path: &'a P,
) -> Option<&'a OsStr> {
use std::os::unix::ffi::OsStrExt;
use memchr::memrchr;
use std::os::unix::ffi::OsStrExt;
let path = path.as_ref().as_os_str().as_bytes();
if path.is_empty() {

273
ignore/src/types.rs → crates/ignore/src/types.rs
View File

@@ -93,220 +93,10 @@ use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
use regex::Regex;
use thread_local::ThreadLocal;
use default_types::DEFAULT_TYPES;
use pathutil::file_name;
use {Error, Match};
const DEFAULT_TYPES: &'static [(&'static str, &'static [&'static str])] = &[
("agda", &["*.agda", "*.lagda"]),
("ats", &["*.ats", "*.dats", "*.sats", "*.hats"]),
("aidl", &["*.aidl"]),
("amake", &["*.mk", "*.bp"]),
("asciidoc", &["*.adoc", "*.asc", "*.asciidoc"]),
("asm", &["*.asm", "*.s", "*.S"]),
("avro", &["*.avdl", "*.avpr", "*.avsc"]),
("awk", &["*.awk"]),
("bazel", &["*.bzl", "WORKSPACE", "BUILD"]),
("bitbake", &["*.bb", "*.bbappend", "*.bbclass", "*.conf", "*.inc"]),
("bzip2", &["*.bz2"]),
("c", &["*.c", "*.h", "*.H", "*.cats"]),
("cabal", &["*.cabal"]),
("cbor", &["*.cbor"]),
("ceylon", &["*.ceylon"]),
("clojure", &["*.clj", "*.cljc", "*.cljs", "*.cljx"]),
("cmake", &["*.cmake", "CMakeLists.txt"]),
("coffeescript", &["*.coffee"]),
("creole", &["*.creole"]),
("config", &["*.cfg", "*.conf", "*.config", "*.ini"]),
("cpp", &[
"*.C", "*.cc", "*.cpp", "*.cxx",
"*.h", "*.H", "*.hh", "*.hpp", "*.hxx", "*.inl",
]),
("crystal", &["Projectfile", "*.cr"]),
("cs", &["*.cs"]),
("csharp", &["*.cs"]),
("cshtml", &["*.cshtml"]),
("css", &["*.css", "*.scss"]),
("csv", &["*.csv"]),
("cython", &["*.pyx"]),
("dart", &["*.dart"]),
("d", &["*.d"]),
("dhall", &["*.dhall"]),
("docker", &["*Dockerfile*"]),
("elisp", &["*.el"]),
("elixir", &["*.ex", "*.eex", "*.exs"]),
("elm", &["*.elm"]),
("erlang", &["*.erl", "*.hrl"]),
("fidl", &["*.fidl"]),
("fish", &["*.fish"]),
("fortran", &[
"*.f", "*.F", "*.f77", "*.F77", "*.pfo",
"*.f90", "*.F90", "*.f95", "*.F95",
]),
("fsharp", &["*.fs", "*.fsx", "*.fsi"]),
("gn", &["*.gn", "*.gni"]),
("go", &["*.go"]),
("gzip", &["*.gz"]),
("groovy", &["*.groovy", "*.gradle"]),
("h", &["*.h", "*.hpp"]),
("hbs", &["*.hbs"]),
("haskell", &["*.hs", "*.lhs", "*.cpphs", "*.c2hs", "*.hsc"]),
("hs", &["*.hs", "*.lhs"]),
("html", &["*.htm", "*.html", "*.ejs"]),
("idris", &["*.idr", "*.lidr"]),
("java", &["*.java", "*.jsp"]),
("jinja", &["*.j2", "*.jinja", "*.jinja2"]),
("js", &[
"*.js", "*.jsx", "*.vue",
]),
("json", &["*.json", "composer.lock"]),
("jsonl", &["*.jsonl"]),
("julia", &["*.jl"]),
("jupyter", &["*.ipynb", "*.jpynb"]),
("jl", &["*.jl"]),
("kotlin", &["*.kt", "*.kts"]),
("less", &["*.less"]),
("license", &[
// General
"COPYING", "COPYING[.-]*",
"COPYRIGHT", "COPYRIGHT[.-]*",
"EULA", "EULA[.-]*",
"licen[cs]e", "licen[cs]e.*",
"LICEN[CS]E", "LICEN[CS]E[.-]*", "*[.-]LICEN[CS]E*",
"NOTICE", "NOTICE[.-]*",
"PATENTS", "PATENTS[.-]*",
"UNLICEN[CS]E", "UNLICEN[CS]E[.-]*",
// GPL (gpl.txt, etc.)
"agpl[.-]*",
"gpl[.-]*",
"lgpl[.-]*",
// Other license-specific (APACHE-2.0.txt, etc.)
"AGPL-*[0-9]*",
"APACHE-*[0-9]*",
"BSD-*[0-9]*",
"CC-BY-*",
"GFDL-*[0-9]*",
"GNU-*[0-9]*",
"GPL-*[0-9]*",
"LGPL-*[0-9]*",
"MIT-*[0-9]*",
"MPL-*[0-9]*",
"OFL-*[0-9]*",
]),
("lisp", &["*.el", "*.jl", "*.lisp", "*.lsp", "*.sc", "*.scm"]),
("log", &["*.log"]),
("lua", &["*.lua"]),
("lzma", &["*.lzma"]),
("lz4", &["*.lz4"]),
("m4", &["*.ac", "*.m4"]),
("make", &[
"gnumakefile", "Gnumakefile", "GNUmakefile",
"makefile", "Makefile",
"*.mk", "*.mak"
]),
("mako", &["*.mako", "*.mao"]),
("markdown", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
("md", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),
("man", &["*.[0-9lnpx]", "*.[0-9][cEFMmpSx]"]),
("matlab", &["*.m"]),
("mk", &["mkfile"]),
("ml", &["*.ml"]),
("msbuild", &[
"*.csproj", "*.fsproj", "*.vcxproj", "*.proj", "*.props", "*.targets"
]),
("nim", &["*.nim"]),
("nix", &["*.nix"]),
("objc", &["*.h", "*.m"]),
("objcpp", &["*.h", "*.mm"]),
("ocaml", &["*.ml", "*.mli", "*.mll", "*.mly"]),
("org", &["*.org"]),
("pascal", &["*.pas", "*.dpr", "*.lpr", "*.pp", "*.inc"]),
("perl", &["*.perl", "*.pl", "*.PL", "*.plh", "*.plx", "*.pm", "*.t"]),
("pdf", &["*.pdf"]),
("php", &["*.php", "*.php3", "*.php4", "*.php5", "*.phtml"]),
("pod", &["*.pod"]),
("protobuf", &["*.proto"]),
("ps", &["*.cdxml", "*.ps1", "*.ps1xml", "*.psd1", "*.psm1"]),
("puppet", &["*.erb", "*.pp", "*.rb"]),
("purs", &["*.purs"]),
("py", &["*.py"]),
("qmake", &["*.pro", "*.pri", "*.prf"]),
("readme", &["README*", "*README"]),
("r", &["*.R", "*.r", "*.Rmd", "*.Rnw"]),
("rdoc", &["*.rdoc"]),
("rst", &["*.rst"]),
("ruby", &["Gemfile", "*.gemspec", ".irbrc", "Rakefile", "*.rb"]),
("rust", &["*.rs"]),
("sass", &["*.sass", "*.scss"]),
("scala", &["*.scala", "*.sbt"]),
("sh", &[
// Portable/misc. init files
".login", ".logout", ".profile", "profile",
// bash-specific init files
".bash_login", "bash_login",
".bash_logout", "bash_logout",
".bash_profile", "bash_profile",
".bashrc", "bashrc", "*.bashrc",
// csh-specific init files
".cshrc", "*.cshrc",
// ksh-specific init files
".kshrc", "*.kshrc",
// tcsh-specific init files
".tcshrc",
// zsh-specific init files
".zshenv", "zshenv",
".zlogin", "zlogin",
".zlogout", "zlogout",
".zprofile", "zprofile",
".zshrc", "zshrc",
// Extensions
"*.bash", "*.csh", "*.ksh", "*.sh", "*.tcsh", "*.zsh",
]),
("smarty", &["*.tpl"]),
("sml", &["*.sml", "*.sig"]),
("soy", &["*.soy"]),
("spark", &["*.spark"]),
("sql", &["*.sql", "*.psql"]),
("stylus", &["*.styl"]),
("sv", &["*.v", "*.vg", "*.sv", "*.svh", "*.h"]),
("svg", &["*.svg"]),
("swift", &["*.swift"]),
("swig", &["*.def", "*.i"]),
("systemd", &[
"*.automount", "*.conf", "*.device", "*.link", "*.mount", "*.path",
"*.scope", "*.service", "*.slice", "*.socket", "*.swap", "*.target",
"*.timer",
]),
("taskpaper", &["*.taskpaper"]),
("tcl", &["*.tcl"]),
("tex", &["*.tex", "*.ltx", "*.cls", "*.sty", "*.bib"]),
("textile", &["*.textile"]),
("tf", &["*.tf"]),
("ts", &["*.ts", "*.tsx"]),
("txt", &["*.txt"]),
("toml", &["*.toml", "Cargo.lock"]),
("twig", &["*.twig"]),
("vala", &["*.vala"]),
("vb", &["*.vb"]),
("verilog", &["*.v", "*.vh", "*.sv", "*.svh"]),
("vhdl", &["*.vhd", "*.vhdl"]),
("vim", &["*.vim"]),
("vimscript", &["*.vim"]),
("wiki", &["*.mediawiki", "*.wiki"]),
("webidl", &["*.idl", "*.webidl", "*.widl"]),
("xml", &["*.xml", "*.xml.dist"]),
("xz", &["*.xz"]),
("yacc", &["*.y"]),
("yaml", &["*.yaml", "*.yml"]),
("zsh", &[
".zshenv", "zshenv",
".zlogin", "zlogin",
".zlogout", "zlogout",
".zprofile", "zprofile",
".zshrc", "zshrc",
"*.zsh",
]),
];
/// Glob represents a single glob in a set of file type definitions.
///
/// There may be more than one glob for a particular file type.
@@ -336,13 +126,23 @@ enum GlobInner<'a> {
which: usize,
/// Whether the selection was negated or not.
negated: bool,
}
},
}
impl<'a> Glob<'a> {
fn unmatched() -> Glob<'a> {
Glob(GlobInner::UnmatchedIgnore)
}
/// Return the file type defintion that matched, if one exists. A file type
/// definition always exists when a specific definition matches a file
/// path.
pub fn file_type_def(&self) -> Option<&FileTypeDef> {
match self {
Glob(GlobInner::UnmatchedIgnore) => None,
Glob(GlobInner::Matched { def, .. }) => Some(def),
}
}
}
/// A single file type definition.
@@ -487,7 +287,7 @@ impl Types {
return Match::None;
}
};
let mut matches = self.matches.get_default().borrow_mut();
let mut matches = self.matches.get_or_default().borrow_mut();
self.set.matches_into(name, &mut *matches);
// The highest precedent match is the last one.
if let Some(&i) = matches.last() {
@@ -526,10 +326,7 @@ impl TypesBuilder {
/// of default type definitions can be added with `add_defaults`, and
/// additional type definitions can be added with `select` and `negate`.
pub fn new() -> TypesBuilder {
TypesBuilder {
types: HashMap::new(),
selections: vec![],
}
TypesBuilder { types: HashMap::new(), selections: vec![] }
}
/// Build the current set of file type definitions *and* selections into
@@ -554,19 +351,18 @@ impl TypesBuilder {
GlobBuilder::new(glob)
.literal_separator(true)
.build()
.map_err(|err| {
Error::Glob {
glob: Some(glob.to_string()),
err: err.kind().to_string(),
}
})?);
.map_err(|err| Error::Glob {
glob: Some(glob.to_string()),
err: err.kind().to_string(),
})?,
);
glob_to_selection.push((isel, iglob));
}
selections.push(selection.clone().map(move |_| def));
}
let set = build_set.build().map_err(|err| {
Error::Glob { glob: None, err: err.to_string() }
})?;
let set = build_set
.build()
.map_err(|err| Error::Glob { glob: None, err: err.to_string() })?;
Ok(Types {
defs: defs,
selections: selections,
@@ -638,9 +434,14 @@ impl TypesBuilder {
return Err(Error::InvalidDefinition);
}
let (key, glob) = (name.to_string(), glob.to_string());
self.types.entry(key).or_insert_with(|| {
FileTypeDef { name: name.to_string(), globs: vec![] }
}).globs.push(glob);
self.types
.entry(key)
.or_insert_with(|| FileTypeDef {
name: name.to_string(),
globs: vec![],
})
.globs
.push(glob);
Ok(())
}
@@ -667,7 +468,10 @@ impl TypesBuilder {
3 => {
let name = parts[0];
let types_string = parts[2];
if name.is_empty() || parts[1] != "include" || types_string.is_empty() {
if name.is_empty()
|| parts[1] != "include"
|| types_string.is_empty()
{
return Err(Error::InvalidDefinition);
}
let types = types_string.split(',');
@@ -677,14 +481,15 @@ impl TypesBuilder {
return Err(Error::InvalidDefinition);
}
for type_name in types {
let globs = self.types.get(type_name).unwrap().globs.clone();
let globs =
self.types.get(type_name).unwrap().globs.clone();
for glob in globs {
self.add(name, &glob)?;
}
}
Ok(())
}
_ => Err(Error::InvalidDefinition)
_ => Err(Error::InvalidDefinition),
}
}
@@ -741,7 +546,7 @@ mod tests {
"rust:*.rs",
"js:*.js",
"foo:*.{rs,foo}",
"combo:include:html,rust"
"combo:include:html,rust",
]
}
@@ -775,7 +580,7 @@ mod tests {
"combo:include:html,python",
// Bad format
"combo:foobar:html,rust",
""
"",
];
for def in bad_defs {
assert!(btypes.add_def(def).is_err());

1043
ignore/src/walk.rs → crates/ignore/src/walk.rs
View File

File diff suppressed because it is too large Load Diff

0
ignore/tests/gitignore_matched_path_or_any_parents_tests.gitignore → crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.gitignore
View File

43
ignore/tests/gitignore_matched_path_or_any_parents_tests.rs → crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.rs
View File

@@ -55,7 +55,6 @@ fn test_files_in_root() {
assert!(m("ROOT/file_root_33").is_none());
}
#[test]
fn test_files_in_deep() {
let gitignore = get_gitignore();
@@ -88,7 +87,6 @@ fn test_files_in_deep() {
assert!(m("ROOT/parent_dir/file_deep_33").is_none());
}
#[test]
fn test_dirs_in_root() {
let gitignore = get_gitignore();
@@ -193,7 +191,6 @@ fn test_dirs_in_root() {
assert!(m("ROOT/dir_root_33/child_dir/file", false).is_ignore());
}
#[test]
fn test_dirs_in_deep() {
let gitignore = get_gitignore();
@@ -205,17 +202,13 @@ fn test_dirs_in_deep() {
assert!(m("ROOT/parent_dir/dir_deep_00", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_00/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_00/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_00/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_00/child_dir/file", false).is_ignore());
// 01
assert!(m("ROOT/parent_dir/dir_deep_01", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_01/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_01/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_01/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_01/child_dir/file", false).is_ignore());
// 02
assert!(m("ROOT/parent_dir/dir_deep_02", true).is_none());
@@ -257,67 +250,51 @@ fn test_dirs_in_deep() {
assert!(m("ROOT/parent_dir/dir_deep_20", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_20/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_20/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_20/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_20/child_dir/file", false).is_ignore());
// 21
assert!(m("ROOT/parent_dir/dir_deep_21", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_21/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_21/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_21/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_21/child_dir/file", false).is_ignore());
// 22
// dir itself doesn't match
assert!(m("ROOT/parent_dir/dir_deep_22", true).is_none());
assert!(m("ROOT/parent_dir/dir_deep_22/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_22/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_22/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_22/child_dir/file", false).is_ignore());
// 23
// dir itself doesn't match
assert!(m("ROOT/parent_dir/dir_deep_23", true).is_none());
assert!(m("ROOT/parent_dir/dir_deep_23/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_23/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_23/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_23/child_dir/file", false).is_ignore());
// 30
assert!(m("ROOT/parent_dir/dir_deep_30", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_30/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_30/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_30/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_30/child_dir/file", false).is_ignore());
// 31
assert!(m("ROOT/parent_dir/dir_deep_31", true).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_31/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_31/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_31/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_31/child_dir/file", false).is_ignore());
// 32
// dir itself doesn't match
assert!(m("ROOT/parent_dir/dir_deep_32", true).is_none());
assert!(m("ROOT/parent_dir/dir_deep_32/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_32/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_32/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_32/child_dir/file", false).is_ignore());
// 33
// dir itself doesn't match
assert!(m("ROOT/parent_dir/dir_deep_33", true).is_none());
assert!(m("ROOT/parent_dir/dir_deep_33/file", false).is_ignore());
assert!(m("ROOT/parent_dir/dir_deep_33/child_dir", true).is_ignore());
assert!(
m("ROOT/parent_dir/dir_deep_33/child_dir/file", false).is_ignore()
);
assert!(m("ROOT/parent_dir/dir_deep_33/child_dir/file", false).is_ignore());
}

10
grep-matcher/Cargo.toml → crates/matcher/Cargo.toml
View File

@@ -1,23 +1,23 @@
[package]
name = "grep-matcher"
version = "0.1.1" #:version
version = "0.1.4" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """
A trait for regular expressions, with a focus on line oriented search.
"""
documentation = "https://docs.rs/grep-matcher"
homepage = "https://github.com/BurntSushi/ripgrep"
repository = "https://github.com/BurntSushi/ripgrep"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/matcher"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/matcher"
readme = "README.md"
keywords = ["regex", "pattern", "trait"]
license = "Unlicense/MIT"
autotests = false
[dependencies]
memchr = "2.0.2"
memchr = "2.1"
[dev-dependencies]
regex = "1.0.5"
regex = "1.1"
[[test]]
name = "integration"

0
grep-printer/LICENSE-MIT → crates/matcher/LICENSE-MIT
View File

0
grep-matcher/README.md → crates/matcher/README.md
View File

0
grep-printer/UNLICENSE → crates/matcher/UNLICENSE
View File

8
grep-matcher/src/interpolate.rs → crates/matcher/src/interpolate.rs
View File

@@ -19,7 +19,7 @@ pub fn interpolate<A, N>(
dst: &mut Vec<u8>,
) where
A: FnMut(usize, &mut Vec<u8>),
N: FnMut(&str) -> Option<usize>
N: FnMut(&str) -> Option<usize>,
{
while !replacement.is_empty() {
match memchr(b'$', replacement) {
@@ -134,14 +134,14 @@ fn find_cap_ref(replacement: &[u8]) -> Option<CaptureRef> {
/// Returns true if and only if the given byte is allowed in a capture name.
fn is_valid_cap_letter(b: &u8) -> bool {
match *b {
b'0' ... b'9' | b'a' ... b'z' | b'A' ... b'Z' | b'_' => true,
b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' | b'_' => true,
_ => false,
}
}
#[cfg(test)]
mod tests {
use super::{CaptureRef, find_cap_ref, interpolate};
use super::{find_cap_ref, interpolate, CaptureRef};
macro_rules! find {
($name:ident, $text:expr) => {
@@ -211,7 +211,7 @@ mod tests {
fn $name() {
assert_eq!($expected, interpolate_string($map, $caps, $hay));
}
}
};
}
interp!(

69
grep-matcher/src/lib.rs → crates/matcher/src/lib.rs
View File

@@ -278,7 +278,7 @@ impl LineTerminator {
}
}
impl Default for LineTerminator {
impl Default for LineTerminator {
#[inline]
fn default() -> LineTerminator {
LineTerminator::byte(b'\n')
@@ -439,7 +439,8 @@ pub trait Captures {
haystack: &[u8],
replacement: &[u8],
dst: &mut Vec<u8>,
) where F: FnMut(&str) -> Option<usize>
) where
F: FnMut(&str) -> Option<usize>,
{
interpolate(
replacement,
@@ -463,12 +464,18 @@ pub struct NoCaptures(());
impl NoCaptures {
/// Create an empty set of capturing groups.
pub fn new() -> NoCaptures { NoCaptures(()) }
pub fn new() -> NoCaptures {
NoCaptures(())
}
}
impl Captures for NoCaptures {
fn len(&self) -> usize { 0 }
fn get(&self, _: usize) -> Option<Match> { None }
fn len(&self) -> usize {
0
}
fn get(&self, _: usize) -> Option<Match> {
None
}
}
/// NoError provides an error type for matchers that never produce errors.
@@ -481,7 +488,9 @@ impl Captures for NoCaptures {
pub struct NoError(());
impl ::std::error::Error for NoError {
fn description(&self) -> &str { "no error" }
fn description(&self) -> &str {
"no error"
}
}
impl fmt::Display for NoError {
@@ -599,10 +608,7 @@ pub trait Matcher {
///
/// The text encoding of `haystack` is not strictly specified. Matchers are
/// advised to assume UTF-8, or at worst, some ASCII compatible encoding.
fn find(
&self,
haystack: &[u8],
) -> Result<Option<Match>, Self::Error> {
fn find(&self, haystack: &[u8]) -> Result<Option<Match>, Self::Error> {
self.find_at(haystack, 0)
}
@@ -614,7 +620,8 @@ pub trait Matcher {
haystack: &[u8],
mut matched: F,
) -> Result<(), Self::Error>
where F: FnMut(Match) -> bool
where
F: FnMut(Match) -> bool,
{
self.try_find_iter(haystack, |m| Ok(matched(m)))
.map(|r: Result<(), ()>| r.unwrap())
@@ -632,7 +639,8 @@ pub trait Matcher {
haystack: &[u8],
mut matched: F,
) -> Result<Result<(), E>, Self::Error>
where F: FnMut(Match) -> Result<bool, E>
where
F: FnMut(Match) -> Result<bool, E>,
{
let mut last_end = 0;
let mut last_match = None;
@@ -690,7 +698,8 @@ pub trait Matcher {
caps: &mut Self::Captures,
mut matched: F,
) -> Result<(), Self::Error>
where F: FnMut(&Self::Captures) -> bool
where
F: FnMut(&Self::Captures) -> bool,
{
self.try_captures_iter(haystack, caps, |caps| Ok(matched(caps)))
.map(|r: Result<(), ()>| r.unwrap())
@@ -709,7 +718,8 @@ pub trait Matcher {
caps: &mut Self::Captures,
mut matched: F,
) -> Result<Result<(), E>, Self::Error>
where F: FnMut(&Self::Captures) -> Result<bool, E>
where
F: FnMut(&Self::Captures) -> Result<bool, E>,
{
let mut last_end = 0;
let mut last_match = None;
@@ -787,7 +797,8 @@ pub trait Matcher {
dst: &mut Vec<u8>,
mut append: F,
) -> Result<(), Self::Error>
where F: FnMut(Match, &mut Vec<u8>) -> bool
where
F: FnMut(Match, &mut Vec<u8>) -> bool,
{
let mut last_match = 0;
self.find_iter(haystack, |m| {
@@ -810,7 +821,8 @@ pub trait Matcher {
dst: &mut Vec<u8>,
mut append: F,
) -> Result<(), Self::Error>
where F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool
where
F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool,
{
let mut last_match = 0;
self.captures_iter(haystack, caps, |caps| {
@@ -1012,10 +1024,7 @@ impl<'a, M: Matcher> Matcher for &'a M {
(*self).capture_count()
}
fn find(
&self,
haystack: &[u8]
) -> Result<Option<Match>, Self::Error> {
fn find(&self, haystack: &[u8]) -> Result<Option<Match>, Self::Error> {
(*self).find(haystack)
}
@@ -1024,7 +1033,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
haystack: &[u8],
matched: F,
) -> Result<(), Self::Error>
where F: FnMut(Match) -> bool
where
F: FnMut(Match) -> bool,
{
(*self).find_iter(haystack, matched)
}
@@ -1034,7 +1044,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
haystack: &[u8],
matched: F,
) -> Result<Result<(), E>, Self::Error>
where F: FnMut(Match) -> Result<bool, E>
where
F: FnMut(Match) -> Result<bool, E>,
{
(*self).try_find_iter(haystack, matched)
}
@@ -1053,7 +1064,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
caps: &mut Self::Captures,
matched: F,
) -> Result<(), Self::Error>
where F: FnMut(&Self::Captures) -> bool
where
F: FnMut(&Self::Captures) -> bool,
{
(*self).captures_iter(haystack, caps, matched)
}
@@ -1064,7 +1076,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
caps: &mut Self::Captures,
matched: F,
) -> Result<Result<(), E>, Self::Error>
where F: FnMut(&Self::Captures) -> Result<bool, E>
where
F: FnMut(&Self::Captures) -> Result<bool, E>,
{
(*self).try_captures_iter(haystack, caps, matched)
}
@@ -1075,7 +1088,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
dst: &mut Vec<u8>,
append: F,
) -> Result<(), Self::Error>
where F: FnMut(Match, &mut Vec<u8>) -> bool
where
F: FnMut(Match, &mut Vec<u8>) -> bool,
{
(*self).replace(haystack, dst, append)
}
@@ -1087,7 +1101,8 @@ impl<'a, M: Matcher> Matcher for &'a M {
dst: &mut Vec<u8>,
append: F,
) -> Result<(), Self::Error>
where F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool
where
F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool,
{
(*self).replace_with_captures(haystack, caps, dst, append)
}
@@ -1099,7 +1114,7 @@ impl<'a, M: Matcher> Matcher for &'a M {
fn is_match_at(
&self,
haystack: &[u8],
at: usize
at: usize,
) -> Result<bool, Self::Error> {
(*self).is_match_at(haystack, at)
}

172
grep-matcher/tests/test_matcher.rs → crates/matcher/tests/test_matcher.rs
View File

@@ -25,18 +25,22 @@ fn find() {
fn find_iter() {
let matcher = matcher(r"(\w+)\s+(\w+)");
let mut matches = vec![];
matcher.find_iter(b"aa bb cc dd", |m| {
matches.push(m);
true
}).unwrap();
matcher
.find_iter(b"aa bb cc dd", |m| {
matches.push(m);
true
})
.unwrap();
assert_eq!(matches, vec![m(0, 5), m(6, 11)]);
// Test that find_iter respects short circuiting.
matches.clear();
matcher.find_iter(b"aa bb cc dd", |m| {
matches.push(m);
false
}).unwrap();
matcher
.find_iter(b"aa bb cc dd", |m| {
matches.push(m);
false
})
.unwrap();
assert_eq!(matches, vec![m(0, 5)]);
}
@@ -47,14 +51,17 @@ fn try_find_iter() {
let matcher = matcher(r"(\w+)\s+(\w+)");
let mut matches = vec![];
let err = matcher.try_find_iter(b"aa bb cc dd", |m| {
if matches.is_empty() {
matches.push(m);
Ok(true)
} else {
Err(MyError)
}
}).unwrap().unwrap_err();
let err = matcher
.try_find_iter(b"aa bb cc dd", |m| {
if matches.is_empty() {
matches.push(m);
Ok(true)
} else {
Err(MyError)
}
})
.unwrap()
.unwrap_err();
assert_eq!(matches, vec![m(0, 5)]);
assert_eq!(err, MyError);
}
@@ -89,28 +96,30 @@ fn captures_iter() {
let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)");
let mut caps = matcher.new_captures().unwrap();
let mut matches = vec![];
matcher.captures_iter(b"aa bb cc dd", &mut caps, |caps| {
matches.push(caps.get(0).unwrap());
matches.push(caps.get(1).unwrap());
matches.push(caps.get(2).unwrap());
true
}).unwrap();
assert_eq!(matches, vec![
m(0, 5), m(0, 2), m(3, 5),
m(6, 11), m(6, 8), m(9, 11),
]);
matcher
.captures_iter(b"aa bb cc dd", &mut caps, |caps| {
matches.push(caps.get(0).unwrap());
matches.push(caps.get(1).unwrap());
matches.push(caps.get(2).unwrap());
true
})
.unwrap();
assert_eq!(
matches,
vec![m(0, 5), m(0, 2), m(3, 5), m(6, 11), m(6, 8), m(9, 11),]
);
// Test that captures_iter respects short circuiting.
matches.clear();
matcher.captures_iter(b"aa bb cc dd", &mut caps, |caps| {
matches.push(caps.get(0).unwrap());
matches.push(caps.get(1).unwrap());
matches.push(caps.get(2).unwrap());
false
}).unwrap();
assert_eq!(matches, vec![
m(0, 5), m(0, 2), m(3, 5),
]);
matcher
.captures_iter(b"aa bb cc dd", &mut caps, |caps| {
matches.push(caps.get(0).unwrap());
matches.push(caps.get(1).unwrap());
matches.push(caps.get(2).unwrap());
false
})
.unwrap();
assert_eq!(matches, vec![m(0, 5), m(0, 2), m(3, 5),]);
}
#[test]
@@ -121,16 +130,19 @@ fn try_captures_iter() {
let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)");
let mut caps = matcher.new_captures().unwrap();
let mut matches = vec![];
let err = matcher.try_captures_iter(b"aa bb cc dd", &mut caps, |caps| {
if matches.is_empty() {
matches.push(caps.get(0).unwrap());
matches.push(caps.get(1).unwrap());
matches.push(caps.get(2).unwrap());
Ok(true)
} else {
Err(MyError)
}
}).unwrap().unwrap_err();
let err = matcher
.try_captures_iter(b"aa bb cc dd", &mut caps, |caps| {
if matches.is_empty() {
matches.push(caps.get(0).unwrap());
matches.push(caps.get(1).unwrap());
matches.push(caps.get(2).unwrap());
Ok(true)
} else {
Err(MyError)
}
})
.unwrap()
.unwrap_err();
assert_eq!(matches, vec![m(0, 5), m(0, 2), m(3, 5)]);
assert_eq!(err, MyError);
}
@@ -150,10 +162,12 @@ fn no_captures() {
assert!(!matcher.captures(b"homer simpson", &mut caps).unwrap());
let mut called = false;
matcher.captures_iter(b"homer simpson", &mut caps, |_| {
called = true;
true
}).unwrap();
matcher
.captures_iter(b"homer simpson", &mut caps, |_| {
called = true;
true
})
.unwrap();
assert!(!called);
}
@@ -161,18 +175,22 @@ fn no_captures() {
fn replace() {
let matcher = matcher(r"(\w+)\s+(\w+)");
let mut dst = vec![];
matcher.replace(b"aa bb cc dd", &mut dst, |_, dst| {
dst.push(b'z');
true
}).unwrap();
matcher
.replace(b"aa bb cc dd", &mut dst, |_, dst| {
dst.push(b'z');
true
})
.unwrap();
assert_eq!(dst, b"z z");
// Test that replacements respect short circuiting.
dst.clear();
matcher.replace(b"aa bb cc dd", &mut dst, |_, dst| {
dst.push(b'z');
false
}).unwrap();
matcher
.replace(b"aa bb cc dd", &mut dst, |_, dst| {
dst.push(b'z');
false
})
.unwrap();
assert_eq!(dst, b"z cc dd");
}
@@ -182,27 +200,31 @@ fn replace_with_captures() {
let haystack = b"aa bb cc dd";
let mut caps = matcher.new_captures().unwrap();
let mut dst = vec![];
matcher.replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| {
caps.interpolate(
|name| matcher.capture_index(name),
haystack,
b"$2 $1",
dst,
);
true
}).unwrap();
matcher
.replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| {
caps.interpolate(
|name| matcher.capture_index(name),
haystack,
b"$2 $1",
dst,
);
true
})
.unwrap();
assert_eq!(dst, b"bb aa dd cc");
// Test that replacements respect short circuiting.
dst.clear();
matcher.replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| {
caps.interpolate(
|name| matcher.capture_index(name),
haystack,
b"$2 $1",
dst,
);
false
}).unwrap();
matcher
.replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| {
caps.interpolate(
|name| matcher.capture_index(name),
haystack,
b"$2 $1",
dst,
);
false
})
.unwrap();
assert_eq!(dst, b"bb aa cc dd");
}

0
grep-matcher/tests/tests.rs → crates/matcher/tests/tests.rs
View File

23
grep-matcher/tests/util.rs → crates/matcher/tests/util.rs
View File

@@ -18,10 +18,7 @@ impl RegexMatcher {
names.insert(name.to_string(), i);
}
}
RegexMatcher {
re: re,
names: names,
}
RegexMatcher { re: re, names: names }
}
}
@@ -31,12 +28,9 @@ impl Matcher for RegexMatcher {
type Captures = RegexCaptures;
type Error = NoError;
fn find_at(
&self,
haystack: &[u8],
at: usize,
) -> Result<Option<Match>> {
Ok(self.re
fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>> {
Ok(self
.re
.find_at(haystack, at)
.map(|m| Match::new(m.start(), m.end())))
}
@@ -75,12 +69,9 @@ impl Matcher for RegexMatcherNoCaps {
type Captures = NoCaptures;
type Error = NoError;
fn find_at(
&self,
haystack: &[u8],
at: usize,
) -> Result<Option<Match>> {
Ok(self.0
fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>> {
Ok(self
.0
.find_at(haystack, at)
.map(|m| Match::new(m.start(), m.end())))
}

10
grep-pcre2/Cargo.toml → crates/pcre2/Cargo.toml
View File

@@ -1,17 +1,17 @@
[package]
name = "grep-pcre2"
version = "0.1.1" #:version
version = "0.1.4" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = """
Use PCRE2 with the 'grep' crate.
"""
documentation = "https://docs.rs/grep-pcre2"
homepage = "https://github.com/BurntSushi/ripgrep"
repository = "https://github.com/BurntSushi/ripgrep"
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/pcre2"
repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/pcre2"
readme = "README.md"
keywords = ["regex", "grep", "pcre", "backreference", "look"]
license = "Unlicense/MIT"
[dependencies]
grep-matcher = { version = "0.1.1", path = "../grep-matcher" }
pcre2 = "0.1.0"
grep-matcher = { version = "0.1.2", path = "../matcher" }
pcre2 = "0.2.0"

Some files were not shown because too many files have changed in this diff Show More