error tracking

DIDONEproject · Oct 2, 2024 · 75d6cf1 · 75d6cf1
2 parents 5b083c2 + 8856fbd
commit 75d6cf1
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 4 deletions.
diff --git a/musif/__main__.py b/musif/__main__.py
@@ -10,8 +10,6 @@
 from musif.logs import perr, pinfo
 from musif.process.processor import DataProcessor
 
-import pandas as pd
-
 def main(
     *paths,
     output_path: str = "musif_features.csv",
@@ -145,6 +143,7 @@ def main(
         config.basic_modules = ["scoring"]
     extract_c.MUSIC21_FILE_EXTENSIONS = extension
     raw_df = FeaturesExtractor(config, limit_files=paths).extract()
+    raw_df = FeaturesExtractor(config, limit_files=paths).extract()
 
     output_path = Path(output_path).with_suffix(".csv")
     # raw_df.to_csv(output_path.with_suffix(".raw.csv"), index=False)
@@ -176,6 +175,7 @@ def main(
     processed_df[string_cols] = processed_df[string_cols].replace("", "-")
     processed_df.to_csv(output_path, index=False)
 
+
 if __name__ == "__main__":
     import fire
 

diff --git a/musif/extract/extract.py b/musif/extract/extract.py
@@ -194,6 +194,10 @@ def find_files(
     else:
         return []
 
+import warnings
+# Suppress all DeprecationWarning messages, particularly for .flat method
+warnings.filterwarnings("ignore", category=DeprecationWarning, module='music21')
+
 
 #  sorted(obj.glob(f"*{extension}"))
 class FeaturesExtractor:
@@ -311,15 +315,18 @@ def _process_corpus(
     ) -> Tuple[List[dict], List[dict]]:
         def process_corpus_par(idx, filename):
             error_files = []
+            errors = []
             try:
                 if self._cfg.window_size is not None:
                     score_features = self._process_score_windows(idx, filename)
                 else:
                     score_features = self._process_score(idx, filename)
             except Exception as e:
-                print('error!')
+                print(f"Error found on {filename}. Saving the filename and error print to {str(self._cfg.output_dir)}/error_files.csv for latter tracking")
                 error_files.append(filename)
-                df = pd.DataFrame(error_files, columns=['ErrorFiles'])
+                errors.append(e)
+                df = pd.DataFrame({'ErrorFiles': error_files,
+                                   'Errors': errors})
                 df.to_csv(str(self._cfg.output_dir)+'/error_files.csv', mode='a', index=False)
                 if self._cfg.ignore_errors:
                     lerr(