Clean up windows performance

Instead of generating random values for the drive and inode counter on
windows we return None instead
This commit is contained in:
andy.boot
2020-02-27 08:24:59 +00:00
committed by Rasmus Halland
parent c30f31c22c
commit 2c58041885
2 changed files with 42 additions and 45 deletions
+35 -28
View File
@@ -85,10 +85,8 @@ pub fn get_dir_tree<P: AsRef<Path>>(
None None
}; };
let mut fake_inode_counter = u64::max_value();
let mut examine_dir_args = ExamineDirMutArsg { let mut examine_dir_args = ExamineDirMutArsg {
data: &mut data, data: &mut data,
fake_inode_counter: &mut fake_inode_counter,
file_count_no_permission: &mut permissions, file_count_no_permission: &mut permissions,
}; };
for b in top_level_names.iter() { for b in top_level_names.iter() {
@@ -127,7 +125,6 @@ pub fn normalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
struct ExamineDirMutArsg<'a> { struct ExamineDirMutArsg<'a> {
data: &'a mut HashMap<PathBuf, u64>, data: &'a mut HashMap<PathBuf, u64>,
file_count_no_permission: &'a mut u64, file_count_no_permission: &'a mut u64,
fake_inode_counter: &'a mut u64,
} }
fn examine_dir<P: AsRef<Path>>( fn examine_dir<P: AsRef<Path>>(
@@ -149,8 +146,7 @@ fn examine_dir<P: AsRef<Path>>(
'entry: for entry in iter { 'entry: for entry in iter {
if let Ok(e) = entry { if let Ok(e) = entry {
let maybe_size_and_inode = let maybe_size_and_inode = get_metadata(&e, apparent_size);
get_metadata(&e, apparent_size, &mut mut_args.fake_inode_counter);
if let Some(dirs) = ignore_directories { if let Some(dirs) = ignore_directories {
let path = e.path(); let path = e.path();
@@ -167,8 +163,9 @@ fn examine_dir<P: AsRef<Path>>(
} }
match maybe_size_and_inode { match maybe_size_and_inode {
Some((size, inode, device)) => { Some(data) => {
if !should_ignore_file(apparent_size, filesystems, &mut inodes, inode, device) { let (size, inode_device) = data;
if !should_ignore_file(apparent_size, filesystems, &mut inodes, inode_device) {
process_file_with_size_and_inode(top_dir, mut_args.data, e, size) process_file_with_size_and_inode(top_dir, mut_args.data, e, size)
} }
} }
@@ -184,24 +181,29 @@ fn should_ignore_file(
apparent_size: bool, apparent_size: bool,
restricted_filesystems: &Option<HashSet<u64>>, restricted_filesystems: &Option<HashSet<u64>>,
inodes: &mut HashSet<(u64, u64)>, inodes: &mut HashSet<(u64, u64)>,
inode: u64, maybe_inode_device: Option<(u64, u64)>,
device: u64,
) -> bool { ) -> bool {
// Ignore files on different devices (if flag applied) match maybe_inode_device {
if let Some(rs) = restricted_filesystems { None => false,
if !rs.contains(&device) { Some(data) => {
return true; let (inode, device) = data;
} // Ignore files on different devices (if flag applied)
} if let Some(rs) = restricted_filesystems {
if !rs.contains(&device) {
return true;
}
}
if !apparent_size { if !apparent_size {
// Ignore files already visited or symlinked // Ignore files already visited or symlinked
if inodes.contains(&(inode, device)) { if inodes.contains(&(inode, device)) {
return true; return true;
}
inodes.insert((inode, device));
}
false
} }
inodes.insert((inode, device));
} }
false
} }
fn process_file_with_size_and_inode<P: AsRef<Path>>( fn process_file_with_size_and_inode<P: AsRef<Path>>(
@@ -365,14 +367,19 @@ mod tests {
let mut files = HashSet::new(); let mut files = HashSet::new();
files.insert((10, 20)); files.insert((10, 20));
assert!(!should_ignore_file(true, &None, &mut files, 0, 0)); assert!(!should_ignore_file(true, &None, &mut files, Some((0, 0))));
// New file is not known it will be inserted to the hashmp and should not be ignored // New file is not known it will be inserted to the hashmp and should not be ignored
assert!(!should_ignore_file(false, &None, &mut files, 11, 12)); assert!(!should_ignore_file(
false,
&None,
&mut files,
Some((11, 12))
));
assert!(files.contains(&(11, 12))); assert!(files.contains(&(11, 12)));
// The same file will be ignored the second time // The same file will be ignored the second time
assert!(should_ignore_file(false, &None, &mut files, 11, 12)); assert!(should_ignore_file(false, &None, &mut files, Some((11, 12))));
} }
#[test] #[test]
@@ -386,11 +393,11 @@ mod tests {
// If we are looking at a different device (disk) and the device flag is set // If we are looking at a different device (disk) and the device flag is set
// then apparent_size is irrelevant - we ignore files on other devices // then apparent_size is irrelevant - we ignore files on other devices
assert!(should_ignore_file(false, &od, &mut files, 11, 12)); assert!(should_ignore_file(false, &od, &mut files, Some((11, 12))));
assert!(should_ignore_file(true, &od, &mut files, 11, 12)); assert!(should_ignore_file(true, &od, &mut files, Some((11, 12))));
// We do not ignore files on the same device // We do not ignore files on the same device
assert!(!should_ignore_file(false, &od, &mut files, 2, 99)); assert!(!should_ignore_file(false, &od, &mut files, Some((2, 99))));
assert!(!should_ignore_file(true, &od, &mut files, 2, 99)); assert!(!should_ignore_file(true, &od, &mut files, Some((2, 99))));
} }
} }
+7 -17
View File
@@ -12,27 +12,19 @@ fn get_block_size() -> u64 {
} }
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
pub fn get_metadata( pub fn get_metadata(d: &DirEntry, use_apparent_size: bool) -> Option<(u64, Option<(u64, u64)>)> {
d: &DirEntry,
use_apparent_size: bool,
_fake_inode_counter: &mut u64,
) -> Option<(u64, u64, u64)> {
use std::os::unix::fs::MetadataExt; use std::os::unix::fs::MetadataExt;
d.metadata.as_ref().unwrap().as_ref().ok().map(|md| { d.metadata.as_ref().unwrap().as_ref().ok().map(|md| {
if use_apparent_size { if use_apparent_size {
(md.len(), md.ino(), md.dev()) (md.len(), Some((md.ino(), md.dev())))
} else { } else {
(md.blocks() * get_block_size(), md.ino(), md.dev()) (md.blocks() * get_block_size(), Some((md.ino(), md.dev())))
} }
}) })
} }
#[cfg(target_family = "windows")] #[cfg(target_family = "windows")]
pub fn get_metadata( pub fn get_metadata(d: &DirEntry, _use_apparent_size: bool) -> Option<(u64, Option<(u64, u64)>)> {
d: &DirEntry,
_use_apparent_size: bool,
fake_inode_counter: &mut u64,
) -> Option<(u64, u64, u64)> {
// On windows opening the file to get size, file ID and volume can be very // On windows opening the file to get size, file ID and volume can be very
// expensive because 1) it causes a few system calls, and more importantly 2) it can cause // expensive because 1) it causes a few system calls, and more importantly 2) it can cause
// windows defender to scan the file. // windows defender to scan the file.
@@ -69,7 +61,7 @@ pub fn get_metadata(
// Consistently opening the file: 30 minutes. // Consistently opening the file: 30 minutes.
// With this optimization: 8 sec. // With this optimization: 8 sec.
fn get_metadata_expensive(d: &DirEntry) -> Option<(u64, u64, u64)> { fn get_metadata_expensive(d: &DirEntry) -> Option<(u64, Option<(u64, u64)>)> {
use winapi_util::file::information; use winapi_util::file::information;
use winapi_util::Handle; use winapi_util::Handle;
@@ -78,8 +70,7 @@ pub fn get_metadata(
Some(( Some((
info.file_size(), info.file_size(),
info.file_index(), Some((info.file_index(), info.volume_serial_number())),
info.volume_serial_number(),
)) ))
} }
@@ -99,8 +90,7 @@ pub fn get_metadata(
|| attr_filtered == FILE_ATTRIBUTE_DIRECTORY || attr_filtered == FILE_ATTRIBUTE_DIRECTORY
|| md.file_attributes() == FILE_ATTRIBUTE_NORMAL || md.file_attributes() == FILE_ATTRIBUTE_NORMAL
{ {
*fake_inode_counter -= 1; Some((md.len(), None))
Some((md.len(), *fake_inode_counter, 0xcafe_cafe_u64))
} else { } else {
get_metadata_expensive(&d) get_metadata_expensive(&d)
} }