quality of life improvements, code refactoring and minor optimizations

.properties

@@ -12,4 +12,5 @@ dataSource.cacheServerConfiguration=true
 dataSource.elideSetAutoCommits=true
 dataSource.maintainTimeStats=false
 ignoreUberJars=false
-numConsumerThreads=10
+numConsumerThreads=10
+totalJars=42

src/main/java/nl/tudelft/cornul11/thesis/database/SignatureDAOImpl.java

@@ -91,7 +91,8 @@ public int insertSignatures(List<Signature> signatures, long jarHash, long jarCr
  }
  }
 
- logger.info(totalRowsInserted + " signature row(s) inserted.");
+ String rowsInserted = "\033[0;32m" + totalRowsInserted + "\033[0m";
+ logger.info(rowsInserted + " signature row(s) inserted.");
  } catch (SQLException e) {
  e.printStackTrace();
  }

src/main/java/nl/tudelft/cornul11/thesis/jarfile/FileAnalyzer.java

@@ -13,19 +13,28 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 public class FileAnalyzer {
  private final List<String> ignoredUberJars = new ArrayList<>();
  private final List<String> insertedLibraries = new ArrayList<>();
+ private int insertedUberJars = 0;
  private final SignatureDAO signatureDao;
  private final Logger logger = LoggerFactory.getLogger(FileAnalyzer.class);
- private final List<Long> uniqueHashes = new ArrayList<>();
+ private final Set<Long> uniqueHashes = new HashSet<>();
  private final ConfigurationLoader config;
+ private AtomicInteger processedJars = new AtomicInteger(0);
+ private int totalJars;
+ private long startTime = System.currentTimeMillis();
+
  public FileAnalyzer(SignatureDAO signatureDao, ConfigurationLoader config) {
  this.config = config;
  this.signatureDao = signatureDao;
+ this.totalJars = config.getTotalJars();
  }
 
  public void printIgnoredUberJars() {
@@ -63,9 +72,9 @@ public void printIgnoredUberJars() {
  }
  }
 
-
  logger.info("Ignored the signatures of " + ignoredUberJars.size() + " uber jars");
- logger.info("Actually inserted " + insertedLibraries.size() + " JARs");
+ logger.info("Inserted the signatures of " + insertedLibraries.size() + " JARs");
+ logger.info("Inserted library information of " + insertedUberJars + " uber JARs");
  }
 
  public int processJarFile(Path jarFilePath) {
@@ -82,6 +91,7 @@ public int processJarFile(Path jarFilePath) {
 
  JarInfoExtractor jarInfoExtractor = new JarInfoExtractor(jarFilePath.toString());
  if (signatures.isEmpty()) { // it's probably an uber-JAR, let's still add it to the db
+ insertedUberJars++;
  return commitLibrary(jarInfoExtractor, jarHash, jarCrc);
  }
 
@@ -94,16 +104,54 @@ public int commitLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long j
  }
 
  public int commitSignatures(List<ClassFileInfo> signatures, JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc) {
- logger.info("Committing signatures for JAR: " + jarInfoExtractor.getArtifactId() + " version: " + jarInfoExtractor.getVersion());
+ logJarCommitment(jarInfoExtractor);
+
  for (ClassFileInfo signature : signatures) {
- if (!uniqueHashes.contains(signature.getHashCode())) {
- uniqueHashes.add(signature.getHashCode());
- }
+ uniqueHashes.add(signature.getHashCode());
  }
- List<Signature> signaturesToInsert = signatures.stream().map(signature -> createSignature(signature, jarInfoExtractor)).collect(Collectors.toList());
- return signatureDao.insertSignatures(signaturesToInsert, jarHash, jarCrc);
+
+ List<Signature> signaturesToInsert = getSignaturesToInsert(signatures, jarInfoExtractor);
+ int insertedRows = signatureDao.insertSignatures(signaturesToInsert, jarHash, jarCrc);
+
+ if (totalJars != -1) {
+ calculateAndLogElapsedTime();
+ }
+ return insertedRows;
+ }
+
+ private void logJarCommitment(JarInfoExtractor jarInfoExtractor) {
+ logger.info(String.format("Committing signatures for JAR: %s version: %s",
+ jarInfoExtractor.getArtifactId(), jarInfoExtractor.getVersion()));
  }
 
+ private List<Signature> getSignaturesToInsert(List<ClassFileInfo> signatures,
+ JarInfoExtractor jarInfoExtractor) {
+ return signatures.stream()
+ .map(signature -> createSignature(signature, jarInfoExtractor))
+ .collect(Collectors.toList());
+ }
+
+ private void calculateAndLogElapsedTime() {
+ int processed = processedJars.incrementAndGet();
+ long elapsedTimeMillis = System.currentTimeMillis() - startTime;
+ double elapsedTimeSec = elapsedTimeMillis / 1000.0;
+ double timePerJarSec = elapsedTimeSec / processed;
+
+ int remainingJars = totalJars - processed;
+ double etaSec = remainingJars * timePerJarSec;
+
+ int etaMin = (int) (etaSec / 60);
+ int etaSecs = (int) (etaSec % 60);
+
+ int etaHour = etaMin / 60;
+ int etaMins = etaMin % 60;
+
+ int etaDays = etaHour / 24;
+ int etaHours = etaHour % 24;
+
+ logger.info(String.format("Done processing %d/%d JARs, progress: \u001B[94m%d%%\u001B[0m, ETA: %d days, %d hours, %d minutes and %d seconds",
+ processed, totalJars, (processed * 100 / totalJars), etaDays, etaHours, etaMins, etaSecs));
+ }
  public void printStats() {
  // TODO: investigate why the number of unique hashes is not constant for a constant given set of JARs
  logger.info("Total number of unique hashes: " + uniqueHashes.size());

src/main/java/nl/tudelft/cornul11/thesis/jarfile/JarProcessor.java

@@ -23,7 +23,8 @@ public JarProcessor(BlockingQueue<Path> queue, FileAnalyzer fileAnalyzer) {
  public void run() {
  try {
  while (true) {
- logger.debug("Waiting for file in the queue, current queue size = " + queue.size());
+ String queueSize = "\033[1;31m" + queue.size() + "\033[0m";
+ logger.debug("Waiting for file in the queue, current queue size = " + queueSize);
  Path file = queue.take(); // this will block if the queue is empty
  if (POISON_PILL.equals(file)) {
  // end-of-stream marker encountered

src/main/java/nl/tudelft/cornul11/thesis/signature/extractor/bytecode/BytecodeUtils.java

@@ -78,15 +78,26 @@ private static String processGenericPart(String genericPart) {
  }
 
  public static String getShortDesc(String desc) {
- int start = desc.indexOf('L');
- while (start != -1) {
- int end = desc.indexOf(';', start);
- if (end == -1) {
+ StringBuilder sb = new StringBuilder();
+ int start = 0;
+ int descLen = desc.length();
+
+ while (start < descLen) {
+ int lIndex = desc.indexOf('L', start);
+ if (lIndex == -1) {
+ sb.append(desc, start, descLen);
+ break;
+ }
+ sb.append(desc, start, lIndex + 1);
+ start = lIndex + 1;
+
+ int semicolonIndex = desc.indexOf(';', start);
+ if (semicolonIndex == -1) {
+ sb.append(desc, start, descLen);
  break;
  }
 
- // Process class name, considering generic types
- String fullName = desc.substring(start + 1, end);
+ String fullName = desc.substring(start, semicolonIndex);
  String shortName;
  int genericStart = fullName.indexOf('<');
  if (genericStart != -1) {
@@ -97,11 +108,11 @@ public static String getShortDesc(String desc) {
  shortName = getShortName(fullName);
  }
 
- desc = desc.substring(0, start + 1) + shortName + desc.substring(end);
- start = desc.indexOf('L', start + 1 + shortName.length() - fullName.length());
+ sb.append(shortName);
+ start = semicolonIndex;
  }
 
- return desc;
+ return sb.toString();
  }
 
  public static String getShortName(String className) {

src/main/java/nl/tudelft/cornul11/thesis/util/ConfigurationLoader.java

@@ -49,4 +49,8 @@ public DatabaseConfig getDatabaseConfig() {
  public int getNumConsumerThreads() {
  return config.getProperty("numConsumerThreads") == null ? 10 : Integer.parseInt(config.getProperty("numConsumerThreads"));
  }
+
+ public int getTotalJars() {
+ return config.getProperty("totalJars") == null ? -1 : Integer.parseInt(config.getProperty("totalJars"));
+ }
 }