better exception handling

.properties

@@ -12,6 +12,6 @@ dataSource.cacheServerConfiguration=true
 dataSource.elideSetAutoCommits=true
 dataSource.maintainTimeStats=false
 dataSource.maximumPoolSize=10 # should be equal to numConsumerThreads
-ignoreUberJars=false
+ignoreUberJarSignatures=true
 numConsumerThreads=10
 totalJars=42

src/main/java/nl/tudelft/cornul11/thesis/commandline/CommandExecutor.java

@@ -58,6 +58,10 @@ public void run() {
  if (fileName != null) {
  JarFrequencyAnalyzer jarFrequencyAnalyzer = new JarFrequencyAnalyzer(signatureDao);
  Map<String, Map<String, Object>> frequencyMap = jarFrequencyAnalyzer.processJar(fileName);
+ if (frequencyMap == null) {
+ System.out.println("Error in processing jar file, ignoring it");
+ return;
+ }
  int totalClassCount = jarFrequencyAnalyzer.getTotalClassCount();
 
  VulnerabilityAnalyzer vulnerabilityAnalyzer = new VulnerabilityAnalyzer(postRequestClient, totalClassCount);

src/main/java/nl/tudelft/cornul11/thesis/database/DatabaseManager.java

@@ -29,6 +29,8 @@ private DatabaseManager(DatabaseConfig config) {
  hikariConfig.addDataSourceProperty("elideSetAutoCommits", config.getElideSetAutoCommits());
  hikariConfig.addDataSourceProperty("maintainTimeStats", config.getMaintainTimeStats());
  hikariConfig.setMaximumPoolSize(Integer.parseInt(config.getMaximumPoolSize()));
+ hikariConfig.setIdleTimeout(60000);
+ hikariConfig.setMaxLifetime(6000000);
 
  ds = new HikariDataSource(hikariConfig);
  logger.info("Connected to the database.");

src/main/java/nl/tudelft/cornul11/thesis/database/SignatureDAO.java

@@ -7,7 +7,7 @@
 import java.util.List;
 
 public interface SignatureDAO {
- int insertLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc);
+ int insertLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc, boolean isBrokenJar);
  int insertSignatures(List<Signature> signatures, long jarHash, long jarCrc);
  List<LibraryMatchInfo> returnTopLibraryMatches(List<Long> hashes);
  void closeConnection();

src/main/java/nl/tudelft/cornul11/thesis/database/SignatureDAOImpl.java

@@ -24,7 +24,7 @@ public SignatureDAOImpl(HikariDataSource ds) {
  }
 
  @Override
- public int insertLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc) {
+ public int insertLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc, boolean isBrokenJar) {
  String insertLibraryQuery = "INSERT INTO libraries (groupId, artifactId, version, hash, crc, isUberJar) VALUES (?, ?, ?, ?, ?, ?)";
 
  executeWithDeadlockRetry(connection -> {
@@ -34,7 +34,7 @@ public int insertLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long j
  libraryStatement.setString(3, jarInfoExtractor.getVersion());
  libraryStatement.setLong(4, jarHash);
  libraryStatement.setLong(5, jarCrc);
- libraryStatement.setBoolean(6, true);
+ libraryStatement.setBoolean(6, !isBrokenJar);
  libraryStatement.executeUpdate();
 
  logger.info("Library row inserted.");

src/main/java/nl/tudelft/cornul11/thesis/jarfile/FileAnalyzer.java

@@ -92,15 +92,15 @@ public int processJarFile(Path jarFilePath) {
  JarInfoExtractor jarInfoExtractor = new JarInfoExtractor(jarFilePath.toString());
  if (signatures.isEmpty()) { // it's probably an uber-JAR, let's still add it to the db
  insertedUberJars++;
- return commitLibrary(jarInfoExtractor, jarHash, jarCrc);
+ return commitLibrary(jarInfoExtractor, jarHash, jarCrc, jarHandler.isBrokenJar());
  }
 
  return commitSignatures(signatures, jarInfoExtractor, jarHash, jarCrc);
  }
 
- public int commitLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc) {
+ public int commitLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc, boolean isBrokenJar) {
  logger.info("Committing library: " + jarInfoExtractor.getArtifactId() + " version: " + jarInfoExtractor.getVersion());
- return signatureDao.insertLibrary(jarInfoExtractor, jarHash, jarCrc);
+ return signatureDao.insertLibrary(jarInfoExtractor, jarHash, jarCrc, isBrokenJar);
  }
 
  public int commitSignatures(List<ClassFileInfo> signatures, JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc) {

src/main/java/nl/tudelft/cornul11/thesis/jarfile/JarHandler.java

@@ -32,6 +32,7 @@ public class JarHandler {
  private final boolean ignoreUberJarSignatures;
  private final CRC32 jarCrc = new CRC32();
  private final long crcValue;
+ private boolean brokenJar = false;
 
  public JarHandler(Path jarFilePath, List<String> ignoredUberJars, List<String> insertedLibraries, ConfigurationLoader config) {
  this.jarFilePath = jarFilePath;
@@ -41,6 +42,10 @@ public JarHandler(Path jarFilePath, List<String> ignoredUberJars, List<String> i
  this.crcValue = this.generateCrc();
  }
 
+ public boolean isBrokenJar() {
+ return brokenJar;
+ }
+
  private long generateCrc() {
  jarCrc.reset();
 
@@ -103,9 +108,11 @@ public List<ClassFileInfo> extractSignatures() {
  return classFileInfos;
  } catch (FileNotFoundException e) {
  // silenced, this is because of the POISON PILL
+ brokenJar = true;
  return new ArrayList<>();
- } catch (IOException | IllegalArgumentException | SecurityException e) {
+ } catch (Exception e) { // goddamn broken JARs
  ignoredUberJars.add(jarFilePath.toString());
+ brokenJar = true;
  logger.error("Error while processing JAR file " + jarFilePath, e);
  return new ArrayList<>();
  }
@@ -173,15 +180,15 @@ private boolean hasMultiplePackages(Path jarFilePath, JarEntry entry, String ini
  return false;
  }
 
- private ClassFileInfo processClassFile(JarEntry entry, JarFile jarFile) throws IOException {
+ private ClassFileInfo processClassFile(JarEntry entry, JarFile jarFile) {
  try (InputStream classFileInputStream = jarFile.getInputStream(entry)) {
  byte[] bytecode = BytecodeUtils.readBytecodeAndCalculateCRCWhenNotAvailable(entry, classFileInputStream);
 
  BytecodeDetails bytecodeDetails = BytecodeParser.extractSignature(bytecode);
  return new ClassFileInfo(entry.getName(), BytecodeUtils.getSignatureHash(bytecodeDetails), entry.getCrc());
- } catch ( Exception e ) {
+ } catch (Exception e) {
  logger.error("Error while processing class file: " + entry.getName(), e);
- throw e;
+ return null;
  }
  }
 

src/main/java/nl/tudelft/cornul11/thesis/jarfile/JarSignatureMapper.java

@@ -48,7 +48,7 @@ public Map<String, Map<String, Object>> inferJarFile(Path jarFilePath) {
  return getTopMatches(classFileInfos, signatureDao);
  } catch (IOException e) {
  logger.error("Error while processing JAR file: " + jarFilePath, e);
- throw new RuntimeException(e);
+ return null;
  }
  }