added better progress and resuming from given last path

Cornul11 · Feb 26, 2024 · e0cc981 · e0cc981
1 parent 560709c
commit e0cc981
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 24 deletions.
diff --git a/src/main/java/nl/tudelft/cornul11/thesis/corpus/commandline/CommandExecutor.java b/src/main/java/nl/tudelft/cornul11/thesis/corpus/commandline/CommandExecutor.java
@@ -81,7 +81,7 @@ public void run() {
  String evaluationDirectory = options.getEvaluationDirectory();
  if (evaluationDirectory != null) {
  JarEvaluator jarEvaluator = new JarEvaluator(signatureDao, evaluationDirectory);
- Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap = jarEvaluator.inferLibrariesFromJars();
+ Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap = jarEvaluator.inferLibrariesFromJars(options.getLastPath());
  jarEvaluator.evaluate(inferredLibrariesMap);
  } else {
  System.out.println("Evaluation directory is required for EVALUATION_MODE");

diff --git a/src/main/java/nl/tudelft/cornul11/thesis/corpus/jarfile/JarEvaluator.java b/src/main/java/nl/tudelft/cornul11/thesis/corpus/jarfile/JarEvaluator.java
@@ -40,7 +40,11 @@ public void evaluate(Map<String, List<JarEvaluator.InferredLibrary>> inferredLib
  }
  }
 
- public Map<String, List<JarEvaluator.InferredLibrary>> inferLibrariesFromJars() {
+ public Map<String, List<JarEvaluator.InferredLibrary>> inferLibrariesFromJars(String resumePath) {
+ boolean resumeProcessing = (resumePath == null || resumePath.isEmpty());
+ Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap = new HashMap<>();
+
+
  try {
  Map<String, List<InferredLibrary>> loadedLibraries = loadInferredLibraries();
  if (!loadedLibraries.isEmpty()) {
@@ -51,40 +55,63 @@ public Map<String, List<JarEvaluator.InferredLibrary>> inferLibrariesFromJars()
  logger.warn("Failed to load inferred libraries from file. Re-inferring them", e);
  }
 
- Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap = new HashMap<>();
-
  File[] projectFolders = getProjectFolders();
  if (projectFolders == null) return inferredLibrariesMap;
 
+ int total = projectFolders.length;
+ logger.info("Inferring libraries from {} projects", total);
+
+ int current = 0;
+ int failedCount = 0;
  for (File projectFolder : projectFolders) {
  String projectName = projectFolder.getName();
  String jarPath = Paths.get(projectFolder.getAbsolutePath(), "target", projectName + "-1.0-SNAPSHOT.jar").toString();
- Path jarFilePath = Path.of(jarPath);
 
+ if (!resumeProcessing) {
+ if (!jarPath.equals(resumePath)) {
+ current++;
+ continue;
+ }
+ resumeProcessing = true;
+ }
+
+ current++;
+
+ Path jarFilePath = Path.of(jarPath);
+ boolean processedSuccessfully = true;
  if (!jarFilePath.toFile().exists()) {
  logger.info("Skipping " + jarPath + " because it does not exist");
- continue;
+ processedSuccessfully = false;
+ } else {
+ List<SignatureDAOImpl.LibraryCandidate> libraryCandidates = jarSignatureMapper.inferJarFileMultithreadedProcess(jarFilePath);
+ if (libraryCandidates == null) {
+ logger.warn("No class file signatures were retrieved from {}", jarFilePath);
+ processedSuccessfully = false;
+ } else {
+ List<JarEvaluator.InferredLibrary> candidateLibraries = libraryCandidates.stream()
+ .map(libraryCandidate -> new JarEvaluator.InferredLibrary(libraryCandidate, jarPath))
+ .collect(Collectors.toList());
+
+ inferredLibrariesMap.put(jarPath, candidateLibraries);
+
+ try {
+ storeInferredLibraries(projectName, candidateLibraries);
+ } catch (IOException e) {
+ logger.error("Failed to store inferred libraries to file", e);
+ }
+ }
  }
 
- List<SignatureDAOImpl.LibraryCandidate> libraryCandidates = jarSignatureMapper.inferJarFileMultithreadedProcess(jarFilePath);
- if (libraryCandidates == null) {
- logger.warn("No class file signatures were retrieved from {}", jarFilePath);
- continue;
+ if (!processedSuccessfully) {
+ failedCount++;
  }
 
- List<JarEvaluator.InferredLibrary> candidateLibraries = libraryCandidates.stream()
- .map(libraryCandidate -> new JarEvaluator.InferredLibrary(libraryCandidate, jarPath))
- .collect(Collectors.toList());
-
-
- inferredLibrariesMap.put(jarPath, candidateLibraries);
+ double percentageCompleted = ((double) current / total) * 100;
+ logger.info("Processed {} ({} failed) out of {} ({}% completed)",
+ current, failedCount, total,
+ String.format("%.2f", percentageCompleted));
  }
 
- try {
- storeInferredLibraries(inferredLibrariesMap);
- } catch (IOException e) {
- logger.error("Failed to store inferred libraries to file", e);
- }
  return inferredLibrariesMap;
  }
 
@@ -146,14 +173,14 @@ private List<JarEvaluator.InferredLibrary> filterLibrariesByThreshold(List<JarEv
  return filteredList;
  }
 
- private void storeInferredLibraries(Map<String, List<JarEvaluator.InferredLibrary>> inferredLibrariesMap) throws IOException {
- Path filePath = Paths.get(evaluationDirectory, "evaluation", "inferredLibraries.json");
+ private void storeInferredLibraries(String projectName, List<JarEvaluator.InferredLibrary> inferredLibraries) throws IOException {
+ Path filePath = Paths.get(evaluationDirectory, "evaluation", projectName + "_inferredLibraries.json");
 
  if (!filePath.getParent().toFile().exists()) {
  filePath.getParent().toFile().mkdirs();
  }
 
- objectMapper.writerWithDefaultPrettyPrinter().writeValue(filePath.toFile(), inferredLibrariesMap);
+ objectMapper.writerWithDefaultPrettyPrinter().writeValue(filePath.toFile(), inferredLibraries);
  logger.info("Stored inferred libraries to {}", filePath);
  }