Merge pull request #541 from devoxx/issue-534

ProjectScannerService refactored into smaller classes + unit tests

Author: stephanj

build.gradle.kts

@@ -75,6 +75,9 @@ dependencies {
     implementation("com.knuddels:jtokkit:1.0.0")
     implementation("org.commonmark:commonmark:0.22.0")
 
+    // GitIgnore Reader
+    implementation("nl.basjes.gitignore:gitignore-reader:1.6.0")
+
     // TDG : Add Log4j dependencies
     implementation("org.apache.logging.log4j:log4j-api:2.22.1")
     implementation("org.apache.logging.log4j:log4j-core:2.22.1")

src/main/java/com/devoxx/genie/service/ProjectContentService.java

@@ -40,14 +40,18 @@ public static ProjectContentService getInstance() {
     public CompletableFuture<ScanContentResult> getProjectContent(Project project,
                                                                   int windowContextMaxTokens,
                                                                   boolean isTokenCalculation) {
-        return ProjectScannerService.getInstance()
-            .scanProject(project, null, windowContextMaxTokens, isTokenCalculation)
-            .thenApply(content -> {
-                if (!isTokenCalculation) {
-                    copyToClipboard(content);
-                }
-                return content;
-            });
+        ProjectScannerService instance = ProjectScannerService.getInstance();
+        ScanContentResult scanContentResult = instance.scanProject(project, null, windowContextMaxTokens, isTokenCalculation);
+        return CompletableFuture.completedFuture(scanContentResult)
+            .thenApply(content ->
+                    getScanContentResult(isTokenCalculation, content));
+    }
+
+    private ScanContentResult getScanContentResult(boolean isTokenCalculation, ScanContentResult content) {
+        if (!isTokenCalculation) {
+            copyToClipboard(content);
+        }
+        return content;
     }
 
     /**
@@ -65,16 +69,18 @@ public CompletableFuture<ScanContentResult> getDirectoryContent(Project project,
                                                                     VirtualFile directory,
                                                                     int tokenLimit,
                                                                     boolean isTokenCalculation) {
-        return ProjectScannerService.getInstance()
-            .scanProject(project, directory, tokenLimit, isTokenCalculation)
-            .thenApply(content -> {
-                if (!isTokenCalculation) {
-                    copyToClipboard(content);
-                }
-                return content;
-            });
+        ProjectScannerService instance = ProjectScannerService.getInstance();
+        ScanContentResult scanContentResult = instance.scanProject(project, directory, tokenLimit, isTokenCalculation);
+        return CompletableFuture.completedFuture(scanContentResult)
+            .thenApply(content ->
+                    getScanContentResult(isTokenCalculation, content));
     }
 
+    /**
+     * Retrieves and processes the content of a specified file within a Project.
+     * @param provider ModelProvider enum value representing the model provider to use
+     * @return Encoding object representing the encoding to use for the specified provider
+     */
     public static Encoding getEncodingForProvider(@NotNull ModelProvider provider) {
         return switch (provider) {
             case OpenAI, Anthropic, Google, AzureOpenAI ->
@@ -88,6 +94,10 @@ public static Encoding getEncodingForProvider(@NotNull ModelProvider provider) {
         };
     }
 
+    /**
+     * Copies the content of a ScanContentResult object to the system clipboard.
+     * @param contentResult ScanContentResult object containing the content to copy
+     */
     private void copyToClipboard(@NotNull ScanContentResult contentResult) {
         Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
         clipboard.setContents(new StringSelection(contentResult.getContent()), null);

src/main/java/com/devoxx/genie/service/TokenCalculationService.java

@@ -40,12 +40,11 @@ public void calculateTokensAndCost(@NotNull Project project,
         if (showCost) {
             showCostAndScanInfo(project, selectedProvider, selectedLanguageModel, contentFuture, listener);
         } else {
-            showOnlyScanInfo(project, directory, selectedProvider, contentFuture, listener);
+            showOnlyScanInfo(directory, selectedProvider, contentFuture, listener);
         }
     }
 
-    private static void showOnlyScanInfo(@NotNull Project project,
-                                         VirtualFile directory,
+    private static void showOnlyScanInfo(VirtualFile directory,
                                          @NotNull ModelProvider selectedProvider,
                                          @NotNull CompletableFuture<ScanContentResult> contentFuture,
                                          TokenCalculationListener listener) {

src/main/java/com/devoxx/genie/service/projectscanner/ContentExtractor.java

@@ -0,0 +1,80 @@
+package com.devoxx.genie.service.projectscanner;
+
+import com.devoxx.genie.ui.settings.DevoxxGenieStateService;
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.application.ReadAction;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.vfs.VirtualFile;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+
+public class ContentExtractor {
+    private static final Logger LOG = Logger.getInstance(ContentExtractor.class.getName());
+
+    /**
+     * Extracts the content of a file and formats it for inclusion in the project scan.
+     */
+    public String extractFileContent(@NotNull VirtualFile file) {
+        StringBuilder content = new StringBuilder();
+        String header = "\n--- " + file.getPath() + " ---\n";
+        content.append(header);
+
+        // Simplified approach - skip the ReadAction complexity for now
+        try {
+            extractFileContentInternal(file, content);
+        } catch (Exception e) {
+            // Just add the error message to the content
+            content.append("Error reading file content: ").append(e.getMessage());
+        }
+
+        return content.toString();
+    }
+
+    /**
+     * Internal implementation to extract file content - separated for easier testing
+     */
+    protected void extractFileContentInternal(VirtualFile file, StringBuilder content) throws IOException {
+        String fileContent;
+
+        try (InputStream is = file.getInputStream();
+             BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
+
+            StringBuilder sb = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                sb.append(line).append('\n');
+            }
+            fileContent = sb.toString();
+        }
+
+        // Process and append the file content
+        content.append(processFileContent(fileContent));
+    }
+
+    public String combineContent(String directoryStructure, String fileContents) {
+        return "Directory Structure:\n" +
+                directoryStructure +
+                "\n\nFile Contents:\n" +
+                fileContents;
+    }
+
+    private String processFileContent(String content) {
+        if (Boolean.TRUE.equals(DevoxxGenieStateService.getInstance().getExcludeJavaDoc())) {
+            return removeJavadoc(content);
+        }
+        return content;
+    }
+
+    private @NotNull String removeJavadoc(String content) {
+        // Remove block comments (which include Javadoc)
+        content = content.replaceAll("/\\*{1,2}[\\s\\S]*?\\*/", "");
+        // Remove single-line comments that start with '///'
+        content = content.replaceAll("^\\s*///.*$", "");
+        return content;
+    }
+}

…anner/UniqueDirectoryScannerService.java …jectscanner/DirectoryScannerService.javasrc/main/java/com/devoxx/genie/service/projectscanner/UniqueDirectoryScannerService.java renamed to src/main/java/com/devoxx/genie/service/projectscanner/DirectoryScannerService.java src/main/java/com/devoxx/genie/service/projectscanner/UniqueDirectoryScannerService.java renamed to src/main/java/com/devoxx/genie/service/projectscanner/DirectoryScannerService.java

@@ -9,7 +9,7 @@
 import java.util.*;
 
 @Getter
-public class UniqueDirectoryScannerService {
+public class DirectoryScannerService {
 
     private final Map<String, VirtualFile> uniqueDirectories = new HashMap<>();