diff --git a/Cargo.lock b/Cargo.lock index 031f2b302e8..ca6f23efff3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3426,7 +3426,7 @@ dependencies = [ [[package]] name = "mithril-common" -version = "0.2.155" +version = "0.2.156" dependencies = [ "anyhow", "async-trait", diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml index 130dac6382d..5314ea24f8f 100644 --- a/mithril-common/Cargo.toml +++ b/mithril-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-common" -version = "0.2.155" +version = "0.2.156" description = "Common types, interfaces, and utilities for Mithril nodes." authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-common/src/digesters/immutable_file.rs b/mithril-common/src/digesters/immutable_file.rs index 810438774bb..02fcdb54d71 100644 --- a/mithril-common/src/digesters/immutable_file.rs +++ b/mithril-common/src/digesters/immutable_file.rs @@ -1,9 +1,9 @@ use crate::entities::{ImmutableFileName, ImmutableFileNumber}; +use crate::digesters::ImmutableFileListingError::MissingImmutableFolder; use digest::{Digest, Output}; use std::{ cmp::Ordering, - ffi::OsStr, fs::File, io, num::ParseIntError, @@ -12,9 +12,23 @@ use std::{ use thiserror::Error; use walkdir::WalkDir; -fn is_immutable(path: &Path) -> bool { - let immutable = OsStr::new("immutable"); - path.iter().any(|component| component == immutable) +const IMMUTABLE_FILE_EXTENSIONS: [&str; 3] = ["chunk", "primary", "secondary"]; + +fn is_immutable(entry: &walkdir::DirEntry) -> bool { + let is_file = entry.file_type().is_file(); + let extension = entry.path().extension().map(|e| e.to_string_lossy()); + + is_file && extension.is_some_and(|e| IMMUTABLE_FILE_EXTENSIONS.contains(&e.as_ref())) +} + +/// Walk the given path and return the first directory named "immutable" it finds +fn find_immutables_dir(path_to_walk: &Path) -> Option { + WalkDir::new(path_to_walk) + .into_iter() + .filter_entry(|e| e.file_type().is_dir()) + .filter_map(|e| e.ok()) + .find(|f| f.file_name() == "immutable") + .map(|e| e.into_path()) } /// Represent an immutable file in a Cardano node database directory @@ -62,6 +76,10 @@ pub enum ImmutableFileListingError { /// Raised when [ImmutableFile::new] fails. #[error("immutable file creation error")] ImmutableFileCreation(#[from] ImmutableFileCreationError), + + /// Raised when the "immutable" folder could not be found in a file structure. + #[error("Couldn't find the 'immutable' folder in '{0:?}'")] + MissingImmutableFolder(PathBuf), } impl ImmutableFile { @@ -116,23 +134,34 @@ impl ImmutableFile { pub fn list_completed_in_dir( dir: &Path, ) -> Result, ImmutableFileListingError> { + let immutable_dir = + find_immutables_dir(dir).ok_or(MissingImmutableFolder(dir.to_path_buf()))?; let mut files: Vec = vec![]; - for path in WalkDir::new(dir) + for path in WalkDir::new(immutable_dir) + .min_depth(1) + .max_depth(1) .into_iter() + .filter_entry(is_immutable) .filter_map(|file| file.ok()) - .map(|f| f.path().to_owned()) { - let metadata = path.metadata()?; - if metadata.is_file() && is_immutable(&path) { - let immutable_file = ImmutableFile::new(path)?; - files.push(immutable_file); - } + let immutable_file = ImmutableFile::new(path.into_path())?; + files.push(immutable_file); } files.sort(); - // @todo: make the skip of the last 'trio' more robust - Ok(files.into_iter().rev().skip(3).rev().collect()) + match files.last() { + // empty list + None => Ok(files), + // filter out the last immutable file(s) + Some(last_file) => { + let last_number = last_file.number; + Ok(files + .into_iter() + .filter(|f| f.number < last_number) + .collect()) + } + } } } @@ -179,6 +208,23 @@ mod tests { } } + fn extract_filenames(immutables: &[ImmutableFile]) -> Vec { + immutables + .iter() + .map(|i| i.path.file_name().unwrap().to_str().unwrap().to_owned()) + .collect() + } + + #[test] + fn list_immutable_file_fail_if_not_in_immutable_dir() { + let target_dir = get_test_dir("list_immutable_file_fail_if_not_in_immutable_dir/invalid"); + let entries = vec![]; + create_fake_files(&target_dir, &entries); + + ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap()) + .expect_err("ImmutableFile::list_in_dir should have Failed"); + } + #[test] fn list_immutable_file_should_skip_last_number() { let target_dir = get_test_dir("list_immutable_file_should_skip_last_number/immutable"); @@ -260,12 +306,59 @@ mod tests { create_fake_files(&target_dir, &entries); let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap()) .expect("ImmutableFile::list_in_dir Failed"); - let immutables_names: Vec = immutables - .into_iter() - .map(|i| i.path.file_name().unwrap().to_str().unwrap().to_owned()) - .collect(); + let immutables_names: Vec = extract_filenames(&immutables); let expected: Vec<&str> = entries.into_iter().rev().skip(3).rev().collect(); assert_eq!(expected, immutables_names); } + + #[test] + fn list_immutable_file_should_work_with_non_immutable_files() { + let target_dir = + get_test_dir("list_immutable_file_should_work_with_non_immutable_files/immutable"); + let entries = vec![ + "123.chunk", + "123.primary", + "123.secondary", + "124.chunk", + "124.primary", + "124.secondary", + "README.md", + "124.secondary.back", + ]; + create_fake_files(&target_dir, &entries); + let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap()) + .expect("ImmutableFile::list_in_dir Failed"); + let immutables_names: Vec = extract_filenames(&immutables); + + let expected: Vec<&str> = entries.into_iter().rev().skip(5).rev().collect(); + assert_eq!(expected, immutables_names); + } + + #[test] + fn list_immutable_file_can_list_incomplete_trio() { + let target_dir = get_test_dir("list_immutable_file_can_list_incomplete_trio/immutable"); + let entries = vec![ + "21.chunk", + "21.primary", + "21.secondary", + "123.chunk", + "123.secondary", + "124.chunk", + "124.primary", + "125.primary", + "125.secondary", + "223.chunk", + "224.primary", + "225.secondary", + "226.chunk", + ]; + create_fake_files(&target_dir, &entries); + let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap()) + .expect("ImmutableFile::list_in_dir Failed"); + let immutables_names: Vec = extract_filenames(&immutables); + + let expected: Vec<&str> = entries.into_iter().rev().skip(1).rev().collect(); + assert_eq!(expected, immutables_names); + } }