Fix: Only create large stack size if enough memory

Small boxes do not have enough memory to create a large stack

Conversely we want a large stack size for large boxes with a very highly
nested directory structure.

Version: New version

.github/workflows/CICD.yml

@@ -81,18 +81,46 @@ jobs:
       matrix:
         job:
           # { os, target, cargo-options, features, use-cross, toolchain }
-          - { os: ubuntu-latest  , target: aarch64-unknown-linux-gnu   , use-cross: use-cross }
+          - {
-          - { os: ubuntu-latest  , target: aarch64-unknown-linux-musl  , use-cross: use-cross }
+              os: ubuntu-latest,
-          - { os: ubuntu-latest  , target: arm-unknown-linux-gnueabihf , use-cross: use-cross }
+              target: aarch64-unknown-linux-gnu,
-          - { os: ubuntu-latest  , target: i686-unknown-linux-gnu      , use-cross: use-cross }
+              use-cross: use-cross,
-          - { os: ubuntu-latest  , target: i686-unknown-linux-musl     , use-cross: use-cross }
+            }
-          - { os: ubuntu-latest  , target: x86_64-unknown-linux-gnu    , use-cross: use-cross }
+          - {
-          - { os: ubuntu-latest  , target: x86_64-unknown-linux-musl   , use-cross: use-cross }
+              os: ubuntu-latest,
-          - { os: macos-latest   , target: x86_64-apple-darwin         }
+              target: aarch64-unknown-linux-musl,
-          - { os: windows-latest , target: i686-pc-windows-gnu         }
+              use-cross: use-cross,
-          - { os: windows-latest , target: i686-pc-windows-msvc        }
+            }
-          - { os: windows-latest , target: x86_64-pc-windows-gnu       } ## !maint: [rivy; 2020-01-21] may break due to rust bug; follow possible solution from GH:rust-lang/rust#47048 (refs: GH:rust-lang/rust#47048 , GH:rust-lang/rust#53454 , GH:bike-barn/hermit#172 )
+          - {
-          - { os: windows-latest , target: x86_64-pc-windows-msvc      }
+              os: ubuntu-latest,
+              target: arm-unknown-linux-gnueabihf,
+              use-cross: use-cross,
+            }
+          - {
+              os: ubuntu-latest,
+              target: i686-unknown-linux-gnu,
+              use-cross: use-cross,
+            }
+          - {
+              os: ubuntu-latest,
+              target: i686-unknown-linux-musl,
+              use-cross: use-cross,
+            }
+          - {
+              os: ubuntu-latest,
+              target: x86_64-unknown-linux-gnu,
+              use-cross: use-cross,
+            }
+          - {
+              os: ubuntu-latest,
+              target: x86_64-unknown-linux-musl,
+              use-cross: use-cross,
+            }
+          - { os: macos-latest, target: x86_64-apple-darwin }
+          - { os: windows-latest, target: i686-pc-windows-gnu }
+          - { os: windows-latest, target: i686-pc-windows-msvc }
+          - { os: windows-latest, target: x86_64-pc-windows-gnu } ## !maint: [rivy; 2020-01-21] may break due to rust bug; follow possible solution from GH:rust-lang/rust#47048 (refs: GH:rust-lang/rust#47048 , GH:rust-lang/rust#53454 , GH:bike-barn/hermit#172 )
+          - { os: windows-latest, target: x86_64-pc-windows-msvc }
     steps:
       - uses: actions/checkout@v1
       - name: Install any prerequisites

Cargo.lock

@@ -68,6 +68,12 @@ dependencies = [
  "regex-automata",
 ]
 
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -118,6 +124,12 @@ dependencies = [
  "toml",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.6"
@@ -197,7 +209,7 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
 
 [[package]]
 name = "du-dust"
-version = "0.8.1"
+version = "0.8.3"
 dependencies = [
  "ansi_term",
  "assert_cmd",
@@ -210,6 +222,7 @@ dependencies = [
  "regex",
  "serde",
  "stfu8",
+ "sysinfo",
  "tempfile",
  "terminal_size",
  "thousands",
@@ -322,6 +335,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "ntapi"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.13.1"
@@ -518,6 +540,23 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "sysinfo"
+version = "0.15.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de94457a09609f33fec5e7fceaf907488967c6c7c75d64da6a7ce6ffdb8b5abd"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "core-foundation-sys",
+ "doc-comment",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "winapi",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.3.0"

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "du-dust"
 description = "A more intuitive version of du"
-version = "0.8.1"
+version = "0.8.3"
 authors = ["bootandy <bootandy@gmail.com>", "nebkor <code@ardent.nebcorp.com>"]
 edition = "2021"
 readme = "README.md"
@@ -39,6 +39,7 @@ regex = "1"
 config-file = "0.2"
 serde = { version = "1.0", features = ["derive"] }
 directories = "4"
+sysinfo = "0.15"
 
 [target.'cfg(windows)'.dependencies]
 winapi-util = "0.1"

README.md

@@ -1,4 +1,3 @@
-
 [![Build Status](https://travis-ci.org/bootandy/dust.svg?branch=master)](https://travis-ci.org/bootandy/dust)
 
 # Dust
@@ -10,35 +9,37 @@ du + rust = dust. Like du but more intuitive.
 Because I want an easy way to see where my disk is being used.
 
 # Demo
+
 ![Example](media/snap.png)
 
 ## Install
 
 #### Cargo <a href="https://repology.org/project/du-dust/versions"><img src="https://repology.org/badge/vertical-allrepos/du-dust.svg" alt="Packaging status" align="right"></a>
 
-* `cargo install du-dust`
+- `cargo install du-dust`
 
 #### 🍺 Homebrew (Mac OS)
 
-* `brew install dust`
+- `brew install dust`
 
 #### 🍺 Homebrew (Linux)
 
-* `brew tap tgotwig/linux-dust && brew install dust`
+- `brew tap tgotwig/linux-dust && brew install dust`
 
 #### [Pacstall](https://github.com/pacstall/pacstall) (Debian/Ubuntu)
 
-* `pacstall -I dust-bin`
+- `pacstall -I dust-bin`
 
 #### Windows:
-* Windows GNU version - works
+
-* Windows MSVC - requires: [VCRUNTIME140.dll](https://docs.microsoft.com/en-gb/cpp/windows/latest-supported-vc-redist?view=msvc-170)
+- Windows GNU version - works
+- Windows MSVC - requires: [VCRUNTIME140.dll](https://docs.microsoft.com/en-gb/cpp/windows/latest-supported-vc-redist?view=msvc-170)
 
 #### Download
 
-* Download Linux/Mac binary from [Releases](https://github.com/bootandy/dust/releases)
+- Download Linux/Mac binary from [Releases](https://github.com/bootandy/dust/releases)
-* unzip file: `tar -xvf _downloaded_file.tar.gz`
+- unzip file: `tar -xvf _downloaded_file.tar.gz`
-* move file to executable path: `sudo mv dust /usr/local/bin/`
+- move file to executable path: `sudo mv dust /usr/local/bin/`
 
 ## Overview
 
@@ -69,18 +70,17 @@ Usage: dust -f (Count files instead of diskspace)
 Usage: dust -t (Group by filetype)
 Usage: dust -z 10M (min-size, Only include files larger than 10M)
 Usage: dust -e regex (Only include files matching this regex (eg dust -e "\.png$" would match png files))
-Usage: dust -v regex (Exculde files matching this regex (eg dust -v "\.png$" would ignore png files))
+Usage: dust -v regex (Exclude files matching this regex (eg dust -v "\.png$" would ignore png files))
 
 ```
 
-
 ## Alternatives
 
-* [NCDU](https://dev.yorhel.nl/ncdu)
+- [NCDU](https://dev.yorhel.nl/ncdu)
-* [dutree](https://github.com/nachoparker/dutree)
+- [dutree](https://github.com/nachoparker/dutree)
-* [dua](https://github.com/Byron/dua-cli/)
+- [dua](https://github.com/Byron/dua-cli/)
-* [pdu](https://github.com/KSXGitHub/parallel-disk-usage)
+- [pdu](https://github.com/KSXGitHub/parallel-disk-usage)
-* [dirstat-rs](https://github.com/scullionw/dirstat-rs)
+- [dirstat-rs](https://github.com/scullionw/dirstat-rs)
-* du -d 1 -h | sort -h
+- du -d 1 -h | sort -h
 
 Note: Apparent-size is calculated slightly differently in dust to gdu. In dust each hard link is counted as using file_length space. In gdu only the first entry is counted.

src/main.rs

@@ -12,6 +12,7 @@ mod utils;
 use crate::cli::build_cli;
 use std::collections::HashSet;
 use std::process;
+use sysinfo::{System, SystemExt};
 
 use self::display::draw_it;
 use clap::Values;
@@ -19,6 +20,7 @@ use config::get_config;
 use dir_walker::{walk_it, WalkData};
 use filter::get_biggest;
 use filter_type::get_all_file_types;
+use rayon::ThreadPoolBuildError;
 use regex::Regex;
 use std::cmp::max;
 use std::path::PathBuf;
@@ -154,11 +156,7 @@ fn main() {
         by_filecount,
         ignore_hidden: config.get_ignore_hidden(&options),
     };
-    // Larger stack size to handle cases with lots of nested directories
+    let _rayon = init_rayon();
-    rayon::ThreadPoolBuilder::new()
-        .stack_size(usize::pow(1024, 3))
-        .build_global()
-        .unwrap();
 
     let iso = config.get_iso(&options);
     let (top_level_nodes, has_errors) = walk_it(simplified_dirs, walk_data);
@@ -191,3 +189,19 @@ fn main() {
         )
     }
 }
+
+fn init_rayon() -> Result<(), ThreadPoolBuildError> {
+    let large_stack = usize::pow(1024, 3);
+    // Warning: Creating System is slow, takes ~ 100ms
+    let s = System::new();
+    let available = s.get_available_memory() * 1024;
+
+    if available > large_stack.try_into().unwrap() {
+        // Larger stack size to handle cases with lots of nested directories
+        rayon::ThreadPoolBuilder::new()
+            .stack_size(large_stack)
+            .build_global()
+    } else {
+        Ok(())
+    }
+}

tests/test_exact_output.rs

@@ -17,7 +17,7 @@ static INIT: Once = Once::new();
 /// Copy to /tmp dir - we assume that the formatting of the /tmp partition
 /// is consistent. If the tests fail your /tmp filesystem probably differs
 fn copy_test_data(dir: &str) {
-    // First remove the existing directory - just incase it is there and has incorrect data
+    // First remove the existing directory - just in case it is there and has incorrect data
     let last_slash = dir.rfind('/').unwrap();
     let last_part_of_dir = dir.chars().skip(last_slash).collect::<String>();
     let _ = Command::new("rm")
@@ -196,7 +196,7 @@ pub fn test_apparent_size() {
 }
 
 fn apparent_size_output() -> Vec<String> {
-    // The apparent directory sizes are too unpredictable and system dependant to try and match
+    // The apparent directory sizes are too unpredictable and system dependent to try and match
     let files = r#"
   0B     ┌── a_file
   6B     ├── hello_file