Skip to content

Commit 5108698

Browse files
stephanjstephanj
authored
Merge pull request #556 from devoxx/feat-555
Improved calc and copy user message
2 parents 6226f59 + b3611a7 commit 5108698

File tree

9 files changed

+209
-52
lines changed

9 files changed

+209
-52
lines changed

‎build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ plugins {
77
}
88

99
group = "com.devoxx.genie"
10-
version = "0.4.21"
10+
version = "0.4.22"
1111

1212
repositories {
1313
mavenCentral()

‎src/main/java/com/devoxx/genie/action/AddDirectoryAction.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,10 +103,29 @@ private void addFilesRecursively(Project project, @NotNull VirtualFile directory
103103
private void copyDirectoryToClipboard(Project project, VirtualFile directory) {
104104
// Because we copy the content to the clipboard, we can set the limit to a high number
105105
CompletableFuture<ScanContentResult> contentFuture = ProjectContentService.getInstance()
106-
.getDirectoryContent(project, directory, 1_000_000, false);
106+
.getDirectoryContent(project, directory, 2_000_000, false);
107107

108-
contentFuture.thenAccept(content ->
109-
NotificationUtil.sendNotification(project, "Directory content added to clipboard: " + directory.getName()));
108+
contentFuture.thenAccept(content -> {
109+
StringBuilder notificationMessage = new StringBuilder("Directory content added to clipboard: ")
110+
.append(directory.getName())
111+
.append("\n");
112+
113+
if (content.getSkippedFileCount() > 0 || content.getSkippedDirectoryCount() > 0) {
114+
notificationMessage.append("(");
115+
if (content.getSkippedFileCount() > 0) {
116+
notificationMessage.append("Skipped: ").append(content.getSkippedFileCount()).append(" files");
117+
}
118+
if (content.getSkippedDirectoryCount() > 0) {
119+
if (content.getSkippedFileCount() > 0) {
120+
notificationMessage.append(" ");
121+
}
122+
notificationMessage.append("Skipped: ").append(content.getSkippedDirectoryCount()).append(" directories");
123+
}
124+
notificationMessage.append(")");
125+
}
126+
127+
NotificationUtil.sendNotification(project, notificationMessage.toString());
128+
});
110129
}
111130

112131
private boolean shouldIncludeFile(@NotNull VirtualFile file, @NotNull DevoxxGenieSettingsService settings) {

‎src/main/java/com/devoxx/genie/model/ScanContentResult.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import java.nio.file.Path;
99
import java.util.ArrayList;
1010
import java.util.List;
11+
import java.util.HashMap;
12+
import java.util.Map;
1113

1214
@AllArgsConstructor
1315
@NoArgsConstructor
@@ -20,7 +22,9 @@ public class ScanContentResult {
2022
private int skippedFileCount;
2123
private int skippedDirectoryCount;
2224
@Getter
23-
private List<Path> files = new ArrayList<>(); // Add this field
25+
private List<Path> files = new ArrayList<>();
26+
@Getter
27+
private Map<String, String> skippedFiles = new HashMap<>();
2428

2529
public void incrementFileCount() {
2630
fileCount++;
@@ -41,4 +45,14 @@ public void addTokenCount(int tokenCount) {
4145
public void addFile(Path file) {
4246
files.add(file);
4347
}
48+
49+
/**
50+
* Add a skipped file with the reason why it was skipped
51+
*
52+
* @param path The path of the skipped file
53+
* @param reason The reason why the file was skipped
54+
*/
55+
public void addSkippedFile(String path, String reason) {
56+
skippedFiles.put(path, reason);
57+
}
4458
}

‎src/main/java/com/devoxx/genie/service/TokenCalculationService.java‎

Lines changed: 93 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,16 @@
44
import com.devoxx.genie.model.LanguageModel;
55
import com.devoxx.genie.model.ScanContentResult;
66
import com.devoxx.genie.model.enumarations.ModelProvider;
7+
import com.devoxx.genie.ui.settings.DevoxxGenieStateService;
78
import com.devoxx.genie.ui.util.NotificationUtil;
89
import com.devoxx.genie.ui.util.WindowContextFormatterUtil;
910
import com.devoxx.genie.util.DefaultLLMSettingsUtil;
1011
import com.intellij.openapi.project.Project;
1112
import com.intellij.openapi.vfs.VirtualFile;
1213
import org.jetbrains.annotations.NotNull;
1314

14-
import java.util.Optional;
15+
import java.io.File;
16+
import java.util.*;
1517
import java.util.concurrent.CompletableFuture;
1618

1719
public class TokenCalculationService {
@@ -49,16 +51,70 @@ private static void showOnlyScanInfo(VirtualFile directory,
4951
@NotNull CompletableFuture<ScanContentResult> contentFuture,
5052
TokenCalculationListener listener) {
5153
contentFuture.thenAccept(result -> {
52-
String message = String.format(
53-
"%s contains %s tokens using the %s tokenizer. " +
54-
"It includes %d files (skipped %d files and %d directories).",
55-
directory != null ? "'" + directory.getName() + "' directory" : "Project",
56-
WindowContextFormatterUtil.format(result.getTokenCount()),
57-
selectedProvider.getName(),
58-
result.getFileCount(),
59-
result.getSkippedFileCount(),
60-
result.getSkippedDirectoryCount());
61-
listener.onTokenCalculationComplete(message);
54+
// Format tokens with K suffix (e.g., 8K)
55+
String formattedTokens = String.format("%.0f", result.getTokenCount() / 1000.0);
56+
57+
// Main message with the requested format
58+
StringBuilder message = new StringBuilder();
59+
if (result.getSkippedDirectoryCount() > 0) {
60+
message.append(String.format(
61+
"'%s' directory contains %sK tokens using the %s tokenizer.\n" +
62+
"It includes %d files, skipped %d files and %d directories.",
63+
directory != null ? directory.getName() : "Project",
64+
formattedTokens,
65+
selectedProvider.getName(),
66+
result.getFileCount(),
67+
result.getSkippedFileCount(),
68+
result.getSkippedDirectoryCount()));
69+
} else {
70+
message.append(String.format(
71+
"'%s' directory contains %sK tokens using the %s tokenizer.\n" +
72+
"It includes %d files, skipped %d files.",
73+
directory != null ? directory.getName() : "Project",
74+
formattedTokens,
75+
selectedProvider.getName(),
76+
result.getFileCount(),
77+
result.getSkippedFileCount()));
78+
}
79+
80+
// Extract and collect skipped file extensions
81+
Set<String> skippedExtensions = new HashSet<>();
82+
Set<String> skippedDirs = new HashSet<>();
83+
84+
if (result.getSkippedFiles() != null && !result.getSkippedFiles().isEmpty()) {
85+
for (Map.Entry<String, String> entry : result.getSkippedFiles().entrySet()) {
86+
String path = entry.getKey();
87+
String reason = entry.getValue();
88+
89+
// Collect skipped extensions
90+
if (path.contains(".")) {
91+
String extension = path.substring(path.lastIndexOf('.') + 1).toLowerCase();
92+
skippedExtensions.add(extension);
93+
}
94+
95+
// Collect skipped directories
96+
if (reason.contains("excluded by settings") || reason.contains("not in project content")) {
97+
File file = new File(path);
98+
String parentDir = file.getParent();
99+
if (parentDir != null) {
100+
String dirName = new File(parentDir).getName();
101+
skippedDirs.add(dirName);
102+
}
103+
}
104+
}
105+
}
106+
107+
// Add skipped file extensions to the message
108+
if (!skippedExtensions.isEmpty()) {
109+
message.append(String.format("\nSkipped files extensions : %s", String.join(", ", skippedExtensions)));
110+
}
111+
112+
// Add skipped directories to the message
113+
if (!skippedDirs.isEmpty() && result.getSkippedDirectoryCount() > 0) {
114+
message.append(String.format("\nSkipped directories : %s", String.join(", ", skippedDirs)));
115+
}
116+
117+
listener.onTokenCalculationComplete(message.toString());
62118
});
63119
}
64120

@@ -135,14 +191,32 @@ private void showInfoForCloudProvider(@NotNull Project project,
135191
}
136192

137193
private @NotNull String getDefaultMessage(@NotNull ScanContentResult scanResult) {
138-
return String.format(
139-
"%s from %d files, skipped " +
140-
(scanResult.getSkippedFileCount() > 0 ? scanResult.getSkippedFileCount() + " files " : "") +
141-
(scanResult.getSkippedFileCount() > 0 && scanResult.getSkippedDirectoryCount() > 0 ? " and " : "") +
142-
(scanResult.getSkippedDirectoryCount() > 0 ? scanResult.getSkippedDirectoryCount() + " directories" : "")
143-
+ ". ",
144-
WindowContextFormatterUtil.format(scanResult.getTokenCount(), "tokens"),
145-
scanResult.getFileCount());
194+
StringBuilder message = new StringBuilder(
195+
WindowContextFormatterUtil.format(scanResult.getTokenCount(), "tokens") +
196+
" from " + scanResult.getFileCount() + " files");
197+
198+
// Add skipped information only if there's anything skipped
199+
if (scanResult.getSkippedFileCount() > 0 || scanResult.getSkippedDirectoryCount() > 0) {
200+
message.append(", skipped ");
201+
202+
// Add skipped files if any
203+
if (scanResult.getSkippedFileCount() > 0) {
204+
message.append(scanResult.getSkippedFileCount()).append(" files");
205+
}
206+
207+
// Add "and" if both files and directories are skipped
208+
if (scanResult.getSkippedFileCount() > 0 && scanResult.getSkippedDirectoryCount() > 0) {
209+
message.append(" and ");
210+
}
211+
212+
// Add skipped directories only if there are any
213+
if (scanResult.getSkippedDirectoryCount() > 0) {
214+
message.append(scanResult.getSkippedDirectoryCount()).append(" directories");
215+
}
216+
}
217+
218+
message.append(". ");
219+
return message.toString();
146220
}
147221

148222
private double calculateCost(int tokenCount, double tokenCost) {

‎src/main/java/com/devoxx/genie/service/projectscanner/FileScanner.java‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.devoxx.genie.service.projectscanner;
22

3+
import com.devoxx.genie.model.ScanContentResult;
34
import com.devoxx.genie.service.DevoxxGenieSettingsService;
45
import com.devoxx.genie.ui.settings.DevoxxGenieStateService;
56
import com.intellij.openapi.diagnostic.Logger;
@@ -114,9 +115,12 @@ public VirtualFile scanProjectModules(Project project) {
114115
*
115116
* @param fileIndex the project file index
116117
* @param directory the directory to scan
118+
* @param scanContentResult the result object to populate
117119
* @return list of files that should be included
118120
*/
119-
public List<VirtualFile> scanDirectory(ProjectFileIndex fileIndex, VirtualFile directory) {
121+
public List<VirtualFile> scanDirectory(ProjectFileIndex fileIndex,
122+
VirtualFile directory,
123+
ScanContentResult scanContentResult) {
120124
List<VirtualFile> relevantFiles = new ArrayList<>();
121125

122126
VfsUtilCore.visitChildrenRecursively(directory, new VirtualFileVisitor<Void>() {
@@ -134,7 +138,9 @@ public boolean visitFile(@NotNull VirtualFile file) {
134138
includedFiles.add(Paths.get(file.getPath()));
135139
} else {
136140
skippedFileCount++;
137-
LOG.debug("Skipping file: " + file.getPath() + " (not in content, excluded by settings, or .gitignore)");
141+
String reason = determineSkipReason(file, fileIndex);
142+
LOG.debug("Skipping file: " + file.getPath() + " (" + reason + ")");
143+
scanContentResult.addSkippedFile(file.getPath(), reason);
138144
}
139145
}
140146
return true;
@@ -143,6 +149,35 @@ public boolean visitFile(@NotNull VirtualFile file) {
143149

144150
return relevantFiles;
145151
}
152+
153+
/**
154+
* Determines the reason why a file was skipped.
155+
*
156+
* @param file the file that was skipped
157+
* @param fileIndex the project file index
158+
* @return the reason for skipping the file
159+
*/
160+
private String determineSkipReason(VirtualFile file, ProjectFileIndex fileIndex) {
161+
if (!fileIndex.isInContent(file)) {
162+
return "not in project content";
163+
}
164+
if (shouldExcludeFile(file)) {
165+
return "excluded by settings or .gitignore";
166+
}
167+
168+
String extension = file.getExtension();
169+
if (extension == null) {
170+
return "no file extension";
171+
}
172+
173+
DevoxxGenieSettingsService settings = DevoxxGenieStateService.getInstance();
174+
List<String> includedExtensions = settings.getIncludedFileExtensions();
175+
if (includedExtensions == null || includedExtensions.isEmpty()) {
176+
return "no file extensions configured for inclusion";
177+
}
178+
179+
return "extension '" + extension.toLowerCase() + "' not in included list";
180+
}
146181

147182
/**
148183
* Generates a source tree representation of a directory.
@@ -219,7 +254,15 @@ public boolean shouldExcludeFile(@NotNull VirtualFile file) {
219254

220255
// Check gitignore if enabled
221256
if (Boolean.TRUE.equals(settings.getUseGitIgnore()) && gitIgnoreFileSet != null) {
222-
return gitIgnoreFileSet.ignoreFile(file.getPath());
257+
try {
258+
// Try to use the gitignore check and catch the specific IllegalArgumentException
259+
return gitIgnoreFileSet.ignoreFile(file.getPath());
260+
} catch (IllegalArgumentException e) {
261+
// If the file is not within the project directory, just ignore the error
262+
// and don't apply gitignore rules to this file
263+
LOG.debug("File outside project directory, skipping gitignore check: " + file.getPath());
264+
return false;
265+
}
223266
}
224267
return false;
225268
}
@@ -240,7 +283,9 @@ public boolean shouldIncludeFile(@NotNull VirtualFile file) {
240283
}
241284

242285
List<String> includedExtensions = settings.getIncludedFileExtensions();
286+
243287
boolean includeFile = includedExtensions != null &&
288+
!includedExtensions.isEmpty() &&
244289
includedExtensions.contains(extension.toLowerCase());
245290

246291
LOG.debug("File: " + file.getPath() + ", Include: " + includeFile + ", Extension: " + extension);

‎src/main/java/com/devoxx/genie/service/projectscanner/ProjectScannerService.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public ScanContentResult scanProject(Project project,
5252
fileScanner.initGitignoreParser(project, startDirectory);
5353
fileScanner.getIncludedFiles().forEach(scanContentResult::addFile);
5454

55-
String content = scanContent(project, startDirectory, windowContextMaxTokens, isTokenCalculation);
55+
String content = scanContent(project, startDirectory, windowContextMaxTokens, isTokenCalculation, scanContentResult);
5656

5757
scanContentResult.setTokenCount(tokenCalculator.calculateTokens(content));
5858
scanContentResult.setContent(content);
@@ -67,7 +67,8 @@ public ScanContentResult scanProject(Project project,
6767
public @NotNull String scanContent(Project project,
6868
VirtualFile startDirectory,
6969
int windowContextMaxTokens,
70-
boolean isTokenCalculation) {
70+
boolean isTokenCalculation,
71+
ScanContentResult scanContentResult) {
7172
// Initialize projectFileIndex if it's null
7273
if (this.projectFileIndex == null) {
7374
this.projectFileIndex = ProjectFileIndex.getInstance(project);
@@ -81,13 +82,13 @@ public ScanContentResult scanProject(Project project,
8182
VirtualFile rootDirectory = fileScanner.scanProjectModules(project);
8283
directoryStructure.append(fileScanner.generateSourceTreeRecursive(rootDirectory, 0));
8384
// Use the stored projectFileIndex instead of getting it again
84-
List<VirtualFile> files = fileScanner.scanDirectory(projectFileIndex, rootDirectory);
85+
List<VirtualFile> files = fileScanner.scanDirectory(projectFileIndex, rootDirectory, scanContentResult);
8586
fileContents = extractAllFileContents(files);
8687
} else if (startDirectory.isDirectory()) {
8788
// Case 2: Directory provided
8889
directoryStructure.append(fileScanner.generateSourceTreeRecursive(startDirectory, 0));
8990
// Use the stored projectFileIndex instead of getting it again
90-
List<VirtualFile> files = fileScanner.scanDirectory(projectFileIndex, startDirectory);
91+
List<VirtualFile> files = fileScanner.scanDirectory(projectFileIndex, startDirectory, scanContentResult);
9192
fileContents = extractAllFileContents(files);
9293
} else {
9394
// Case 3: Single file provided

‎src/main/resources/META-INF/plugin.xml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,14 @@
3535
]]></description>
3636

3737
<change-notes><![CDATA[
38+
<h2>V0.4.22</h2>
39+
<UL>
40+
<LI>Feat #555 : Improve "Calc Tokens... " and "Copy directory... " user messages by @stephanj</LI>
41+
</UL>
3842
<h2>V0.4.21</h2>
3943
<UL>
40-
<LI>Fix #553 integration with LMStudio by @mydeveloperplanet</LI>
41-
<LI>Feat ##550 Allow extra text to be added after custom prompts by @mydeveloperplanet</LI>
44+
<LI>Fix #553 : integration with LMStudio by @mydeveloperplanet</LI>
45+
<LI>Feat ##550 : Allow extra text to be added after custom prompts by @mydeveloperplanet</LI>
4246
</UL>
4347
<h2>V0.4.20</h2>
4448
<UL>
@@ -196,7 +200,7 @@
196200
<UL>
197201
<LI>Feature #327: Test Driven Generation integration</LI>
198202
<LI>Fix #324: Replace invocations of SwingUtilities.invokeLater()</LI>
199-
<LI>Fix #236: Refactoring large class ActionButtonsPanel<LI>
203+
<LI>Fix #236: Refactoring large class ActionButtonsPanel</LI>
200204
<LI>Fix #329: Refactored the calc token cost</LI>
201205
<LI>Fix #333: Use LangChain4j new model names</LI>
202206
<LI>Fix #336: Cost Calculation is consistent now between calculation from directory and calculation from DevoxxGenie panel</LI>
@@ -428,7 +432,7 @@
428432
<UL>
429433
<LI>Feat #63: Support for Gemini Pro 1.5 & Flash using API Keys</LI>
430434
<LI>Fix for incorrect LLM selection</LI>
431-
<LI>Fix for chat response styling issue<LI>
435+
<LI>Fix for chat response styling issue</LI>
432436
<LI>Fix prompt context issue</LI>
433437
</UL>
434438
<h2>v0.1.7</h2>

‎src/main/resources/application.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=0.4.20
1+
version=0.4.22

0 commit comments

Comments
 (0)