From 2c58041885232949d4a946de340dd875c0310ec8 Mon Sep 17 00:00:00 2001 From: "andy.boot" Date: Thu, 27 Feb 2020 08:24:59 +0000 Subject: [PATCH] Clean up windows performance Instead of generating random values for the drive and inode counter on windows we return None instead --- src/utils/mod.rs | 63 ++++++++++++++++++++++++------------------- src/utils/platform.rs | 24 +++++------------ 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 2261bd5..f4342bb 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -85,10 +85,8 @@ pub fn get_dir_tree>( None }; - let mut fake_inode_counter = u64::max_value(); let mut examine_dir_args = ExamineDirMutArsg { data: &mut data, - fake_inode_counter: &mut fake_inode_counter, file_count_no_permission: &mut permissions, }; for b in top_level_names.iter() { @@ -127,7 +125,6 @@ pub fn normalize_path>(path: P) -> PathBuf { struct ExamineDirMutArsg<'a> { data: &'a mut HashMap, file_count_no_permission: &'a mut u64, - fake_inode_counter: &'a mut u64, } fn examine_dir>( @@ -149,8 +146,7 @@ fn examine_dir>( 'entry: for entry in iter { if let Ok(e) = entry { - let maybe_size_and_inode = - get_metadata(&e, apparent_size, &mut mut_args.fake_inode_counter); + let maybe_size_and_inode = get_metadata(&e, apparent_size); if let Some(dirs) = ignore_directories { let path = e.path(); @@ -167,8 +163,9 @@ fn examine_dir>( } match maybe_size_and_inode { - Some((size, inode, device)) => { - if !should_ignore_file(apparent_size, filesystems, &mut inodes, inode, device) { + Some(data) => { + let (size, inode_device) = data; + if !should_ignore_file(apparent_size, filesystems, &mut inodes, inode_device) { process_file_with_size_and_inode(top_dir, mut_args.data, e, size) } } @@ -184,24 +181,29 @@ fn should_ignore_file( apparent_size: bool, restricted_filesystems: &Option>, inodes: &mut HashSet<(u64, u64)>, - inode: u64, - device: u64, + maybe_inode_device: Option<(u64, u64)>, ) -> bool { - // Ignore files on different devices (if flag applied) - if let Some(rs) = restricted_filesystems { - if !rs.contains(&device) { - return true; - } - } + match maybe_inode_device { + None => false, + Some(data) => { + let (inode, device) = data; + // Ignore files on different devices (if flag applied) + if let Some(rs) = restricted_filesystems { + if !rs.contains(&device) { + return true; + } + } - if !apparent_size { - // Ignore files already visited or symlinked - if inodes.contains(&(inode, device)) { - return true; + if !apparent_size { + // Ignore files already visited or symlinked + if inodes.contains(&(inode, device)) { + return true; + } + inodes.insert((inode, device)); + } + false } - inodes.insert((inode, device)); } - false } fn process_file_with_size_and_inode>( @@ -365,14 +367,19 @@ mod tests { let mut files = HashSet::new(); files.insert((10, 20)); - assert!(!should_ignore_file(true, &None, &mut files, 0, 0)); + assert!(!should_ignore_file(true, &None, &mut files, Some((0, 0)))); // New file is not known it will be inserted to the hashmp and should not be ignored - assert!(!should_ignore_file(false, &None, &mut files, 11, 12)); + assert!(!should_ignore_file( + false, + &None, + &mut files, + Some((11, 12)) + )); assert!(files.contains(&(11, 12))); // The same file will be ignored the second time - assert!(should_ignore_file(false, &None, &mut files, 11, 12)); + assert!(should_ignore_file(false, &None, &mut files, Some((11, 12)))); } #[test] @@ -386,11 +393,11 @@ mod tests { // If we are looking at a different device (disk) and the device flag is set // then apparent_size is irrelevant - we ignore files on other devices - assert!(should_ignore_file(false, &od, &mut files, 11, 12)); - assert!(should_ignore_file(true, &od, &mut files, 11, 12)); + assert!(should_ignore_file(false, &od, &mut files, Some((11, 12)))); + assert!(should_ignore_file(true, &od, &mut files, Some((11, 12)))); // We do not ignore files on the same device - assert!(!should_ignore_file(false, &od, &mut files, 2, 99)); - assert!(!should_ignore_file(true, &od, &mut files, 2, 99)); + assert!(!should_ignore_file(false, &od, &mut files, Some((2, 99)))); + assert!(!should_ignore_file(true, &od, &mut files, Some((2, 99)))); } } diff --git a/src/utils/platform.rs b/src/utils/platform.rs index 8ef25a5..121b643 100644 --- a/src/utils/platform.rs +++ b/src/utils/platform.rs @@ -12,27 +12,19 @@ fn get_block_size() -> u64 { } #[cfg(target_family = "unix")] -pub fn get_metadata( - d: &DirEntry, - use_apparent_size: bool, - _fake_inode_counter: &mut u64, -) -> Option<(u64, u64, u64)> { +pub fn get_metadata(d: &DirEntry, use_apparent_size: bool) -> Option<(u64, Option<(u64, u64)>)> { use std::os::unix::fs::MetadataExt; d.metadata.as_ref().unwrap().as_ref().ok().map(|md| { if use_apparent_size { - (md.len(), md.ino(), md.dev()) + (md.len(), Some((md.ino(), md.dev()))) } else { - (md.blocks() * get_block_size(), md.ino(), md.dev()) + (md.blocks() * get_block_size(), Some((md.ino(), md.dev()))) } }) } #[cfg(target_family = "windows")] -pub fn get_metadata( - d: &DirEntry, - _use_apparent_size: bool, - fake_inode_counter: &mut u64, -) -> Option<(u64, u64, u64)> { +pub fn get_metadata(d: &DirEntry, _use_apparent_size: bool) -> Option<(u64, Option<(u64, u64)>)> { // On windows opening the file to get size, file ID and volume can be very // expensive because 1) it causes a few system calls, and more importantly 2) it can cause // windows defender to scan the file. @@ -69,7 +61,7 @@ pub fn get_metadata( // Consistently opening the file: 30 minutes. // With this optimization: 8 sec. - fn get_metadata_expensive(d: &DirEntry) -> Option<(u64, u64, u64)> { + fn get_metadata_expensive(d: &DirEntry) -> Option<(u64, Option<(u64, u64)>)> { use winapi_util::file::information; use winapi_util::Handle; @@ -78,8 +70,7 @@ pub fn get_metadata( Some(( info.file_size(), - info.file_index(), - info.volume_serial_number(), + Some((info.file_index(), info.volume_serial_number())), )) } @@ -99,8 +90,7 @@ pub fn get_metadata( || attr_filtered == FILE_ATTRIBUTE_DIRECTORY || md.file_attributes() == FILE_ATTRIBUTE_NORMAL { - *fake_inode_counter -= 1; - Some((md.len(), *fake_inode_counter, 0xcafe_cafe_u64)) + Some((md.len(), None)) } else { get_metadata_expensive(&d) }