feat: Custom stack size

If a user reports a crash from stacksize - we can try different stack
memory sizes to try and figure out the cause.

Do not try and initialise rayon on 32 bit builds

Fix rayon initialisation, may not have been called before due to badly
formed closure
This commit is contained in:
andy.boot
2023-12-11 23:15:31 +00:00
parent c8b61d2f46
commit e8c7990a17
10 changed files with 69 additions and 21 deletions
+8
View File
@@ -186,6 +186,14 @@ pub fn build_cli() -> Command {
.action(clap::ArgAction::SetTrue)
.help("Only files will be displayed. (Finds your largest files)"),
)
.arg(
Arg::new("stack_size")
.short('S')
.long("stack-size")
.num_args(1)
.value_parser(value_parser!(usize))
.help("Specify memory to use as stack size - use if you see: 'fatal runtime error: stack overflow' (default low memory=1048576, high memory=1073741824)"),
)
.arg(Arg::new("params").num_args(1..)
.value_parser(value_parser!(String)))
}
+9
View File
@@ -26,6 +26,7 @@ pub struct Config {
pub disable_progress: Option<bool>,
pub depth: Option<usize>,
pub bars_on_right: Option<bool>,
pub stack_size: Option<usize>,
}
impl Config {
@@ -95,6 +96,14 @@ impl Config {
pub fn get_bars_on_right(&self, options: &ArgMatches) -> bool {
Some(true) == self.bars_on_right || options.get_flag("bars_on_right")
}
pub fn get_custom_stack_size(&self, options: &ArgMatches) -> Option<usize> {
let from_cmd_line = options.get_one::<usize>("stack_size");
if from_cmd_line.is_none() {
self.stack_size
} else {
from_cmd_line.copied()
}
}
}
fn convert_min_size(input: &str, iso: bool) -> Option<usize> {
+30 -18
View File
@@ -27,7 +27,6 @@ use config::get_config;
use dir_walker::walk_it;
use filter::get_biggest;
use filter_type::get_all_file_types;
use rayon::ThreadPoolBuildError;
use regex::Regex;
use std::cmp::max;
use std::path::PathBuf;
@@ -177,11 +176,8 @@ fn main() {
follow_links,
progress_data: indicator.data.clone(),
};
let result = panic::catch_unwind(|| init_rayon);
if result.is_err() {
eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1")
}
let stack_size = config.get_custom_stack_size(&options);
init_rayon(&stack_size);
let top_level_nodes = walk_it(simplified_dirs, walk_data);
@@ -227,18 +223,34 @@ fn main() {
}
}
fn init_rayon() -> Result<(), ThreadPoolBuildError> {
let large_stack = usize::pow(1024, 3);
let mut s = System::new();
s.refresh_memory();
let available = s.available_memory();
fn init_rayon(stack_size: &Option<usize>) {
// Rayon seems to raise this error on 32-bit builds
// The global thread pool has not been initialized.: ThreadPoolBuildError { kind: GlobalPoolAlreadyInitialized }
if cfg!(target_pointer_width = "64") {
let result = panic::catch_unwind(|| {
match stack_size {
Some(n) => rayon::ThreadPoolBuilder::new()
.stack_size(*n)
.build_global(),
None => {
let large_stack = usize::pow(1024, 3);
let mut s = System::new();
s.refresh_memory();
let available = s.available_memory();
if available > large_stack.try_into().unwrap() {
// Larger stack size to handle cases with lots of nested directories
rayon::ThreadPoolBuilder::new()
.stack_size(large_stack)
.build_global()
} else {
rayon::ThreadPoolBuilder::new().build_global()
if available > large_stack.try_into().unwrap() {
// Larger stack size to handle cases with lots of nested directories
rayon::ThreadPoolBuilder::new()
.stack_size(large_stack)
.build_global()
} else {
rayon::ThreadPoolBuilder::new().build_global()
}
}
}
});
if result.is_err() {
eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1")
}
}
}