Add parallel recursive directory iterator.

This adds a new walk type in the `ignore` crate, `WalkParallel`, which
provides a way for recursively iterating over a set of paths in parallel
while respecting various ignore rules.

The API is a bit strange, as a closure producing a closure isn't
something one often sees, but it does seem to work well.

This also allowed us to simplify much of the worker logic in ripgrep
proper, where MultiWorker is now gone.

ignore/src/lib.rs

@@ -44,6 +44,7 @@ for result in WalkBuilder::new("./").hidden(false).build() {
 See the documentation for `WalkBuilder` for many other options.
 */
 
+extern crate crossbeam;
 extern crate globset;
 #[macro_use]
 extern crate lazy_static;
@@ -61,7 +62,7 @@ use std::fmt;
 use std::io;
 use std::path::{Path, PathBuf};
 
-pub use walk::{DirEntry, Walk, WalkBuilder};
+pub use walk::{DirEntry, Walk, WalkBuilder, WalkParallel, WalkState};
 
 mod dir;
 pub mod gitignore;
@@ -80,6 +81,12 @@ pub enum Error {
     WithLineNumber { line: u64, err: Box<Error> },
     /// An error associated with a particular file path.
     WithPath { path: PathBuf, err: Box<Error> },
+    /// An error associated with a particular directory depth when recursively
+    /// walking a directory.
+    WithDepth { depth: usize, err: Box<Error> },
+    /// An error that occurs when a file loop is detected when traversing
+    /// symbolic links.
+    Loop { ancestor: PathBuf, child: PathBuf },
     /// An error that occurs when doing I/O, such as reading an ignore file.
     Io(io::Error),
     /// An error that occurs when trying to parse a glob.
@@ -101,6 +108,7 @@ impl Error {
             Error::Partial(_) => true,
             Error::WithLineNumber { ref err, .. } => err.is_partial(),
             Error::WithPath { ref err, .. } => err.is_partial(),
+            Error::WithDepth { ref err, .. } => err.is_partial(),
             _ => false,
         }
     }
@@ -111,6 +119,8 @@ impl Error {
             Error::Partial(ref errs) => errs.len() == 1 && errs[0].is_io(),
             Error::WithLineNumber { ref err, .. } => err.is_io(),
             Error::WithPath { ref err, .. } => err.is_io(),
+            Error::WithDepth { ref err, .. } => err.is_io(),
+            Error::Loop { .. } => false,
             Error::Io(_) => true,
             Error::Glob(_) => false,
             Error::UnrecognizedFileType(_) => false,
@@ -118,6 +128,16 @@ impl Error {
         }
     }
 
+    /// Returns a depth associated with recursively walking a directory (if
+    /// this error was generated from a recursive directory iterator).
+    pub fn depth(&self) -> Option<usize> {
+        match *self {
+            Error::WithPath { ref err, .. } => err.depth(),
+            Error::WithDepth { depth, .. } => Some(depth),
+            _ => None,
+        }
+    }
+
     /// Turn an error into a tagged error with the given file path.
     fn with_path<P: AsRef<Path>>(self, path: P) -> Error {
         Error::WithPath {
@@ -126,6 +146,14 @@ impl Error {
         }
     }
 
+    /// Turn an error into a tagged error with the given depth.
+    fn with_depth(self, depth: usize) -> Error {
+        Error::WithDepth {
+            depth: depth,
+            err: Box::new(self),
+        }
+    }
+
     /// Turn an error into a tagged error with the given file path and line
     /// number. If path is empty, then it is omitted from the error.
     fn tagged<P: AsRef<Path>>(self, path: P, lineno: u64) -> Error {
@@ -146,6 +174,8 @@ impl error::Error for Error {
             Error::Partial(_) => "partial error",
             Error::WithLineNumber { ref err, .. } => err.description(),
             Error::WithPath { ref err, .. } => err.description(),
+            Error::WithDepth { ref err, .. } => err.description(),
+            Error::Loop { .. } => "file system loop found",
             Error::Io(ref err) => err.description(),
             Error::Glob(ref msg) => msg,
             Error::UnrecognizedFileType(_) => "unrecognized file type",
@@ -168,6 +198,12 @@ impl fmt::Display for Error {
             Error::WithPath { ref path, ref err } => {
                 write!(f, "{}: {}", path.display(), err)
             }
+            Error::WithDepth { ref err, .. } => err.fmt(f),
+            Error::Loop { ref ancestor, ref child } => {
+                write!(f, "File system loop found: \
+                           {} points to an ancestor {}",
+                          child.display(), ancestor.display())
+            }
             Error::Io(ref err) => err.fmt(f),
             Error::Glob(ref msg) => write!(f, "{}", msg),
             Error::UnrecognizedFileType(ref ty) => {
@@ -187,6 +223,30 @@ impl From<io::Error> for Error {
     }
 }
 
+impl From<walkdir::Error> for Error {
+    fn from(err: walkdir::Error) -> Error {
+        let depth = err.depth();
+        if let (Some(anc), Some(child)) = (err.loop_ancestor(), err.path()) {
+            return Error::WithDepth {
+                depth: depth,
+                err: Box::new(Error::Loop {
+                    ancestor: anc.to_path_buf(),
+                    child: child.to_path_buf(),
+                }),
+            };
+        }
+        let path = err.path().map(|p| p.to_path_buf());
+        let mut ig_err = Error::Io(io::Error::from(err));
+        if let Some(path) = path {
+            ig_err = Error::WithPath {
+                path: path,
+                err: Box::new(ig_err),
+            };
+        }
+        ig_err
+    }
+}
+
 #[derive(Debug, Default)]
 struct PartialErrorBuilder(Vec<Error>);