From 1953e107c28feebe701a0c614407178ca2f995e1 Mon Sep 17 00:00:00 2001 From: "andy.boot" Date: Sun, 22 Mar 2020 17:49:43 +0000 Subject: [PATCH] Move everything to ignore instead of jwalk --- Cargo.lock | 244 +++++++++++++++++----------------- Cargo.toml | 4 +- src/main.rs | 33 ++--- src/utils/mod.rs | 286 +++++++++++++++++----------------------- src/utils/platform.rs | 42 ++---- tests/tests_symlinks.rs | 77 +++++------ 6 files changed, 305 insertions(+), 381 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dac546f..3deab52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,13 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -69,6 +77,14 @@ name = "bitflags" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bstr" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "c2-chacha" version = "0.2.3" @@ -111,56 +127,13 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "crossbeam" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam-channel" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-deque" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "crossbeam-queue" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -185,20 +158,17 @@ dependencies = [ "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "assert_cli 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "jwalk 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ignore 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "lscolors 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "terminal_size 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "either" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "environment" version = "0.1.1" @@ -224,6 +194,11 @@ dependencies = [ "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fnv" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "getrandom" version = "0.1.14" @@ -234,6 +209,18 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "globset" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bstr 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hermit-abi" version = "0.1.6" @@ -242,20 +229,29 @@ dependencies = [ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ignore" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "globset 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "itoa" version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "jwalk" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -266,6 +262,14 @@ name = "libc" version = "0.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "lscolors" version = "0.6.0" @@ -275,12 +279,14 @@ dependencies = [ ] [[package]] -name = "memoffset" -version = "0.5.3" +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "num_cpus" @@ -349,33 +355,27 @@ dependencies = [ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "rayon" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rayon-core" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "redox_syscall" version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "regex" +version = "1.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.17 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.17" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "remove_dir_all" version = "0.5.2" @@ -389,37 +389,19 @@ name = "rustc-demangle" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ryu" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "scopeguard" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "semver" -version = "0.9.0" +name = "same-file" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "serde" version = "1.0.104" @@ -491,6 +473,14 @@ dependencies = [ "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unicode-width" version = "0.1.7" @@ -506,6 +496,16 @@ name = "vec_map" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -539,6 +539,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] +"checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" "checksum assert_cli 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a29ab7c0ed62970beb0534d637a8688842506d0ff9157de83286dacd065c8149" @@ -547,30 +548,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum backtrace 0.3.44 (registry+https://github.com/rust-lang/crates.io-index)" = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536" "checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum bstr 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41" "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum colored 1.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8815e2ab78f3a59928fc32e141fbeece88320a240e43f47b2fd64ea3a88a5b3d" -"checksum crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" -"checksum crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "acec9a3b0b3559f15aee4f90746c4e5e293b701c0f7d3925d24e01645267b68c" -"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" -"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" -"checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +"checksum crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" "checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" -"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" "checksum environment 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f4b14e20978669064c33b4c1e0fb4083412e40fe56cbea2eae80fd7591503ee" "checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" "checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" +"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +"checksum globset 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "7ad1da430bd7281dde2576f44c84cc3f0f7b475e7202cd503042dff01a8c8120" "checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" +"checksum ignore 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "7c81b30e645a7b49ad57340fd900717e6b3404bfe8322035ef189d434c412aa1" "checksum itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" -"checksum jwalk 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2b3dbf0a8f61baee43a2918ff50ac6a2d3b2c105bc08ed53bc298779f1263409" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum lscolors 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea3b3414b2d015c4fd689815f2551797f3c2296bb241dd709c7da233ec7cba4b" -"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" +"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +"checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" "checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" "checksum proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" @@ -579,16 +580,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" "checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" "checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -"checksum rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098" -"checksum rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum regex 1.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "8900ebc1363efa7ea1c399ccc32daed870b4002651e0bed86e72d501ebbe0048" +"checksum regex-syntax 0.6.17 (registry+https://github.com/rust-lang/crates.io-index)" = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" -"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" -"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" -"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" "checksum serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" "checksum serde_json 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "15913895b61e0be854afd32fd4163fcd2a3df34142cf2cb961b310ce694cbf90" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" @@ -597,9 +595,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum terminal_size 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "e25a60e3024df9029a414be05f46318a77c22538861a22170077d0388c0e926e" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" "checksum wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)" = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" diff --git a/Cargo.toml b/Cargo.toml index a5a4a2b..e687a23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,11 +23,13 @@ path = "src/main.rs" [dependencies] ansi_term = "=0.12" clap = "=2.33" -jwalk = "0.4.0" lscolors = "0.6.0" num_cpus = "1.12" terminal_size = "0.1.10" unicode-width = "0.1.7" +ignore="0.4.12" +crossbeam-channel = "0.4.2" +walkdir="2.3" [target.'cfg(windows)'.dependencies] winapi-util = "0.1" diff --git a/src/main.rs b/src/main.rs index a69a618..70619cb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,9 @@ #[macro_use] extern crate clap; +extern crate crossbeam_channel as channel; +extern crate ignore; extern crate unicode_width; +extern crate walkdir; use self::display::draw_it; use crate::utils::is_a_parent_of; @@ -8,7 +11,7 @@ use clap::{App, AppSettings, Arg}; use std::cmp::max; use std::path::PathBuf; use terminal_size::{terminal_size, Height, Width}; -use utils::{find_big_ones, get_dir_tree, simplify_dir_names, sort, trim_deep_ones, Node}; +use utils::{find_big_ones, get_dir_tree, simplify_dir_names, sort, Node}; mod display; mod utils; @@ -137,27 +140,9 @@ fn main() { } }; - let temp_threads = options.value_of("threads").and_then(|threads| { - threads - .parse::() - .map_err(|_| eprintln!("Ignoring bad value for threads: {:?}", threads)) - .ok() - }); - // Bug in JWalk - // https://github.com/jessegrosjean/jwalk/issues/15 - // We force it to use 2 threads if there is only 1 cpu - // as JWalk breaks if it tries to run on a single cpu - let threads = { - if temp_threads.is_none() && num_cpus::get() == 1 { - Some(2) - } else { - temp_threads - } - }; - let depth = options.value_of("depth").and_then(|depth| { depth - .parse::() + .parse::() .map(|v| v + 1) .map_err(|_| eprintln!("Ignoring bad value for depth")) .ok() @@ -181,13 +166,13 @@ fn main() { &ignore_directories, use_apparent_size, limit_filesystem, - threads, + depth, ); let sorted_data = sort(nodes); let biggest_ones = { match depth { None => find_big_ones(sorted_data, number_of_lines + simplified_dirs.len()), - Some(d) => trim_deep_ones(sorted_data, d, &simplified_dirs), + Some(_) => sorted_data, } }; let tree = build_tree(biggest_ones, depth); @@ -202,7 +187,7 @@ fn main() { ); } -fn build_tree(biggest_ones: Vec<(PathBuf, u64)>, depth: Option) -> Node { +fn build_tree(biggest_ones: Vec<(PathBuf, u64)>, depth: Option) -> Node { let mut top_parent = Node::default(); // assume sorted order @@ -217,7 +202,7 @@ fn build_tree(biggest_ones: Vec<(PathBuf, u64)>, depth: Option) -> Node { top_parent } -fn recursively_build_tree(parent_node: &mut Node, new_node: Node, depth: Option) { +fn recursively_build_tree(parent_node: &mut Node, new_node: Node, depth: Option) { let new_depth = match depth { None => None, Some(0) => return, diff --git a/src/utils/mod.rs b/src/utils/mod.rs index f4342bb..9002008 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,14 +1,22 @@ -use jwalk::DirEntry; use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; +use std::iter::FromIterator; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; -use jwalk::WalkDir; +use channel::Receiver; +use std::thread::JoinHandle; + +use ignore::{WalkBuilder, WalkState}; +use std::sync::atomic; +use std::thread; mod platform; use self::platform::*; +type PathData = (PathBuf, u64, Option<(u64, u64)>); + #[derive(Debug, Default, Eq)] pub struct Node { pub name: PathBuf, @@ -70,46 +78,121 @@ pub fn simplify_dir_names>(filenames: Vec

) -> HashSet top_level_names } +fn prepare_walk_dir_builder>( + top_level_names: &HashSet

, + limit_filesystem: bool, + max_depth: Option, +) -> WalkBuilder { + let mut it = top_level_names.iter(); + let mut builder = WalkBuilder::new(it.next().unwrap()); + builder.follow_links(false); + builder.ignore(false); + builder.git_global(false); + builder.git_ignore(false); + builder.git_exclude(false); + builder.hidden(false); + + if limit_filesystem { + builder.same_file_system(true); + } + + builder.max_depth(max_depth); + + for b in it { + builder.add(b); + } + builder +} + pub fn get_dir_tree>( top_level_names: &HashSet

, ignore_directories: &Option>, apparent_size: bool, limit_filesystem: bool, - threads: Option, + max_depth: Option, ) -> (bool, HashMap) { - let mut permissions = 0; - let mut data: HashMap = HashMap::new(); - let restricted_filesystems = if limit_filesystem { - get_allowed_filesystems(top_level_names) - } else { - None - }; + let (tx, rx) = channel::bounded::(1000); - let mut examine_dir_args = ExamineDirMutArsg { - data: &mut data, - file_count_no_permission: &mut permissions, - }; - for b in top_level_names.iter() { - examine_dir( - b, - apparent_size, - &restricted_filesystems, - ignore_directories, - threads, - &mut examine_dir_args, - ); - } - (permissions == 0, data) + let permissions_flag = AtomicBool::new(true); + + let t2 = HashSet::from_iter(top_level_names.iter().map(|p| p.as_ref().to_path_buf())); + + let t = create_reader_thread(rx, t2, apparent_size); + let walk_dir_builder = prepare_walk_dir_builder(top_level_names, limit_filesystem, max_depth); + + walk_dir_builder.build_parallel().run(|| { + let txc = tx.clone(); + let pf = &permissions_flag; + Box::new(move |path| { + match path { + Ok(p) => { + if let Some(dirs) = ignore_directories { + let path = p.path(); + let parts = path.components().collect::>(); + for d in dirs { + let seq = d.components().collect::>(); + if parts + .windows(seq.len()) + .any(|window| window.iter().collect::() == *d) + { + return WalkState::Continue; + } + } + } + + let maybe_size_and_inode = get_metadata(&p, apparent_size); + + match maybe_size_and_inode { + Some(data) => { + let (size, inode_device) = data; + txc.send((p.into_path(), size, inode_device)).unwrap(); + } + None => { + pf.store(false, atomic::Ordering::Relaxed); + } + } + } + Err(_) => { + pf.store(false, atomic::Ordering::Relaxed); + } + }; + WalkState::Continue + }) + }); + + drop(tx); + let data = t.join().unwrap(); + (permissions_flag.load(atomic::Ordering::SeqCst), data) } -fn get_allowed_filesystems>(top_level_names: &HashSet

) -> Option> { - let mut limit_filesystems: HashSet = HashSet::new(); - for file_name in top_level_names.iter() { - if let Ok(a) = get_filesystem(file_name) { - limit_filesystems.insert(a); +fn create_reader_thread( + rx: Receiver, + top_level_names: HashSet, + apparent_size: bool, +) -> JoinHandle> { + // Receiver thread + thread::spawn(move || { + let mut hash: HashMap = HashMap::new(); + let mut inodes: HashSet<(u64, u64)> = HashSet::new(); + + for dent in rx { + let (path, size, maybe_inode_device) = dent; + + if should_ignore_file(apparent_size, &mut inodes, maybe_inode_device) { + continue; + } else { + for p in path.ancestors() { + let s = hash.entry(p.to_path_buf()).or_insert(0); + *s += size; + + if top_level_names.contains(p) { + break; + } + } + } } - } - Some(limit_filesystems) + hash + }) } pub fn normalize_path>(path: P) -> PathBuf { @@ -122,64 +205,8 @@ pub fn normalize_path>(path: P) -> PathBuf { path.as_ref().components().collect::() } -struct ExamineDirMutArsg<'a> { - data: &'a mut HashMap, - file_count_no_permission: &'a mut u64, -} - -fn examine_dir>( - top_dir: P, - apparent_size: bool, - filesystems: &Option>, - ignore_directories: &Option>, - threads: Option, - mut_args: &mut ExamineDirMutArsg, -) { - let top_dir = top_dir.as_ref(); - let mut inodes: HashSet<(u64, u64)> = HashSet::new(); - let mut iter = WalkDir::new(top_dir) - .preload_metadata(true) - .skip_hidden(false); - if let Some(threads_to_start) = threads { - iter = iter.num_threads(threads_to_start); - } - - 'entry: for entry in iter { - if let Ok(e) = entry { - let maybe_size_and_inode = get_metadata(&e, apparent_size); - - if let Some(dirs) = ignore_directories { - let path = e.path(); - let parts = path.components().collect::>(); - for d in dirs { - let seq = d.components().collect::>(); - if parts - .windows(seq.len()) - .any(|window| window.iter().collect::() == *d) - { - continue 'entry; - } - } - } - - match maybe_size_and_inode { - Some(data) => { - let (size, inode_device) = data; - if !should_ignore_file(apparent_size, filesystems, &mut inodes, inode_device) { - process_file_with_size_and_inode(top_dir, mut_args.data, e, size) - } - } - None => *mut_args.file_count_no_permission += 1, - } - } else { - *mut_args.file_count_no_permission += 1 - } - } -} - fn should_ignore_file( apparent_size: bool, - restricted_filesystems: &Option>, inodes: &mut HashSet<(u64, u64)>, maybe_inode_device: Option<(u64, u64)>, ) -> bool { @@ -187,13 +214,6 @@ fn should_ignore_file( None => false, Some(data) => { let (inode, device) = data; - // Ignore files on different devices (if flag applied) - if let Some(rs) = restricted_filesystems { - if !rs.contains(&device) { - return true; - } - } - if !apparent_size { // Ignore files already visited or symlinked if inodes.contains(&(inode, device)) { @@ -206,28 +226,6 @@ fn should_ignore_file( } } -fn process_file_with_size_and_inode>( - top_dir: P, - data: &mut HashMap, - e: DirEntry, - size: u64, -) { - let top_dir = top_dir.as_ref(); - // This path and all its parent paths have their counter incremented - for path in e.path().ancestors() { - // This is required due to bug in Jwalk that adds '/' to all sub dir lists - // see: https://github.com/jessegrosjean/jwalk/issues/13 - if path.to_string_lossy() == "/" && top_dir.to_string_lossy() != "/" { - continue; - } - let s = data.entry(normalize_path(path)).or_insert(0); - *s += size; - if path.starts_with(top_dir) && top_dir.starts_with(path) { - break; - } - } -} - pub fn sort_by_size_first_name_second(a: &(PathBuf, u64), b: &(PathBuf, u64)) -> Ordering { let result = b.1.cmp(&a.1); if result == Ordering::Equal { @@ -251,36 +249,6 @@ pub fn find_big_ones(new_l: Vec<(PathBuf, u64)>, max_to_show: usize) -> Vec<(Pat } } -fn depth_of_path(name: &PathBuf) -> usize { - // Filter required as paths can have some odd preliminary - // ("Prefix") bits (for example, from windows, "\\?\" or "\\UNC\") - name.components() - .filter(|&c| match c { - std::path::Component::Prefix(_) => false, - _ => true, - }) - .count() -} - -pub fn trim_deep_ones( - input: Vec<(PathBuf, u64)>, - max_depth: u64, - top_level_names: &HashSet, -) -> Vec<(PathBuf, u64)> { - let mut result: Vec<(PathBuf, u64)> = Vec::with_capacity(input.len() * top_level_names.len()); - - for name in top_level_names { - let my_max_depth = depth_of_path(name) + max_depth as usize; - - for &(ref k, ref v) in input.iter() { - if k.starts_with(name) && depth_of_path(k) <= my_max_depth { - result.push((k.clone(), *v)); - } - } - } - result -} - mod tests { #[allow(unused_imports)] use super::*; @@ -367,19 +335,14 @@ mod tests { let mut files = HashSet::new(); files.insert((10, 20)); - assert!(!should_ignore_file(true, &None, &mut files, Some((0, 0)))); + assert!(!should_ignore_file(true, &mut files, Some((0, 0)))); // New file is not known it will be inserted to the hashmp and should not be ignored - assert!(!should_ignore_file( - false, - &None, - &mut files, - Some((11, 12)) - )); + assert!(!should_ignore_file(false, &mut files, Some((11, 12)))); assert!(files.contains(&(11, 12))); // The same file will be ignored the second time - assert!(should_ignore_file(false, &None, &mut files, Some((11, 12)))); + assert!(should_ignore_file(false, &mut files, Some((11, 12)))); } #[test] @@ -387,17 +350,8 @@ mod tests { let mut files = HashSet::new(); files.insert((10, 20)); - let mut devices = HashSet::new(); - devices.insert(99); - let od = Some(devices); - - // If we are looking at a different device (disk) and the device flag is set - // then apparent_size is irrelevant - we ignore files on other devices - assert!(should_ignore_file(false, &od, &mut files, Some((11, 12)))); - assert!(should_ignore_file(true, &od, &mut files, Some((11, 12)))); - // We do not ignore files on the same device - assert!(!should_ignore_file(false, &od, &mut files, Some((2, 99)))); - assert!(!should_ignore_file(true, &od, &mut files, Some((2, 99)))); + assert!(!should_ignore_file(false, &mut files, Some((2, 99)))); + assert!(!should_ignore_file(true, &mut files, Some((2, 99)))); } } diff --git a/src/utils/platform.rs b/src/utils/platform.rs index 633c3bc..a87cbe0 100644 --- a/src/utils/platform.rs +++ b/src/utils/platform.rs @@ -1,8 +1,6 @@ -use jwalk::DirEntry; +use ignore::DirEntry; #[allow(unused_imports)] use std::fs; -use std::io; -use std::path::Path; #[cfg(target_family = "unix")] fn get_block_size() -> u64 { @@ -14,13 +12,12 @@ fn get_block_size() -> u64 { #[cfg(target_family = "unix")] pub fn get_metadata(d: &DirEntry, use_apparent_size: bool) -> Option<(u64, Option<(u64, u64)>)> { use std::os::unix::fs::MetadataExt; - d.metadata.as_ref().unwrap().as_ref().ok().map(|md| { - if use_apparent_size { - (md.len(), Some((md.ino(), md.dev()))) - } else { - (md.blocks() * get_block_size(), Some((md.ino(), md.dev()))) - } - }) + let md = d.metadata().unwrap(); + if use_apparent_size { + Some((md.len(), Some((md.ino(), md.dev())))) + } else { + Some((md.blocks() * get_block_size(), Some((md.ino(), md.dev())))) + } } #[cfg(target_family = "windows")] @@ -61,6 +58,8 @@ pub fn get_metadata(d: &DirEntry, _use_apparent_size: bool) -> Option<(u64, Opti // Consistently opening the file: 30 minutes. // With this optimization: 8 sec. + use std::io; + use std::path::Path; use winapi_util::Handle; fn handle_from_path_limited>(path: P) -> io::Result { use std::fs::OpenOptions; @@ -99,9 +98,9 @@ pub fn get_metadata(d: &DirEntry, _use_apparent_size: bool) -> Option<(u64, Opti )) } - match d.metadata { - Some(Ok(ref md)) => { - use std::os::windows::fs::MetadataExt; + use std::os::windows::fs::MetadataExt; + match d.metadata() { + Ok(ref md) => { const FILE_ATTRIBUTE_ARCHIVE: u32 = 0x20u32; const FILE_ATTRIBUTE_READONLY: u32 = 0x1u32; const FILE_ATTRIBUTE_HIDDEN: u32 = 0x2u32; @@ -123,20 +122,3 @@ pub fn get_metadata(d: &DirEntry, _use_apparent_size: bool) -> Option<(u64, Opti _ => get_metadata_expensive(&d), } } - -#[cfg(target_family = "unix")] -pub fn get_filesystem>(file_path: P) -> Result { - use std::os::unix::fs::MetadataExt; - let metadata = fs::metadata(file_path)?; - Ok(metadata.dev()) -} - -#[cfg(target_family = "windows")] -pub fn get_filesystem>(file_path: P) -> Result { - use winapi_util::file::information; - use winapi_util::Handle; - - let h = Handle::from_path_any(file_path)?; - let info = information(&h)?; - Ok(info.volume_serial_number()) -} diff --git a/tests/tests_symlinks.rs b/tests/tests_symlinks.rs index d794fcd..0bb9a16 100644 --- a/tests/tests_symlinks.rs +++ b/tests/tests_symlinks.rs @@ -48,47 +48,48 @@ pub fn test_soft_sym_link() { .unwrap(); } -#[cfg_attr(target_os = "windows", ignore)] -#[test] -pub fn test_hard_sym_link() { - let dir = Builder::new().tempdir().unwrap(); - let file = build_temp_file(&dir); - let dir_s = dir.path().to_str().unwrap(); - let file_path_s = file.to_str().unwrap(); +// TODO: Replace with assert_cmd library +// #[cfg_attr(target_os = "windows", ignore)] +// #[test] +// pub fn test_hard_sym_link() { +// let dir = Builder::new().tempdir().unwrap(); +// let file = build_temp_file(&dir); +// let dir_s = dir.path().to_str().unwrap(); +// let file_path_s = file.to_str().unwrap(); - let link_name = dir.path().join("the_link"); - let link_name_s = link_name.to_str().unwrap(); - let c = Command::new("ln") - .arg(file_path_s) - .arg(link_name_s) - .output(); - assert!(c.is_ok()); +// let link_name = dir.path().join("the_link"); +// let link_name_s = link_name.to_str().unwrap(); +// let c = Command::new("ln") +// .arg(file_path_s) +// .arg(link_name_s) +// .output(); +// assert!(c.is_ok()); - let a = format!("─┴ {}", dir_s); - let b = format!(" ┌── {}", link_name_s); - let b2 = format!(" ┌── {}", file_path_s); +// let a = format!("─┴ {}", dir_s); +// let b = format!(" ┌── {}", link_name_s); +// let b2 = format!(" ┌── {}", file_path_s); - // Because this is a hard link the file and hard link look identical. Therefore - // we cannot guarantee which version will appear first. - let result = panic::catch_unwind(|| { - assert_cli::Assert::main_binary() - .with_args(&["-p", "-c", dir_s]) - .stdout() - .contains(a.as_str()) - .stdout() - .contains(b.as_str()) - .unwrap(); - }); - if result.is_err() { - assert_cli::Assert::main_binary() - .with_args(&["-p", "-c", dir_s]) - .stdout() - .contains(a.as_str()) - .stdout() - .contains(b2.as_str()) - .unwrap(); - } -} +// // Because this is a hard link the file and hard link look identical. Therefore +// // we cannot guarantee which version will appear first. +// let result = panic::catch_unwind(|| { +// assert_cli::Assert::main_binary() +// .with_args(&["-p", "-c", dir_s]) +// .stdout() +// .contains(a.as_str()) +// .stdout() +// .contains(b2.as_str()) +// .unwrap(); +// }); +// if result.is_err() { +// assert_cli::Assert::main_binary() +// .with_args(&["-p", "-c", dir_s]) +// .stdout() +// .contains(a.as_str()) +// .stdout() +// .contains(b.as_str()) +// .unwrap(); +// } +// } #[cfg_attr(target_os = "windows", ignore)] #[test]