added simple maven-apache-shade config extraction for the oracle

Dockerfile-app

@@ -13,4 +13,4 @@ WORKDIR /usr/src/app
 COPY --from=build /usr/src/app/target/thesis-1.0-SNAPSHOT.jar /usr/src/app
 COPY --from=build /usr/src/app/target/dependency /usr/src/app/libs
 ENV TZ=Europe/Amsterdam
-ENTRYPOINT java -Xmx8g -cp /usr/src/app/libs/*:/usr/src/app/thesis-1.0-SNAPSHOT.jar nl.tudelft.cornul11.thesis.MainApp -m CORPUS_GEN_MODE -d ${DIRECTORY}
+ENTRYPOINT java -Xmx8g -cp /usr/src/app/libs/*:/usr/src/app/thesis-1.0-SNAPSHOT.jar nl.tudelft.cornul11.thesis.corpus.MainApp -m CORPUS_GEN_MODE -d ${DIRECTORY}

pom.xml

@@ -101,6 +101,17 @@
  <version>5.0.1</version>
  </dependency>
 
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-model</artifactId>
+ <version>3.9.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.5.1</version>
+ </dependency>
 
  </dependencies>
 
@@ -123,7 +134,7 @@
  <configuration>
  <archive>
  <manifest>
- <mainClass>nl.tudelft.cornul11.thesis.MainApp</mainClass>
+ <mainClass>nl.tudelft.cornul11.thesis.corpus.MainApp</mainClass>
  </manifest>
  </archive>
  </configuration>

src/main/java/nl/tudelft/cornul11/thesis/MainApp.java → src/main/java/nl/tudelft/cornul11/thesis/corpus/MainApp.java

@@ -1,4 +1,4 @@
-package nl.tudelft.cornul11.thesis;
+package nl.tudelft.cornul11.thesis.corpus;
 
 import nl.tudelft.cornul11.thesis.corpus.api.PostRequestClient;
 import nl.tudelft.cornul11.thesis.corpus.commandline.CommandExecutor;

src/main/java/nl/tudelft/cornul11/thesis/corpus/database/DatabaseManager.java

@@ -56,10 +56,63 @@ public SignatureDAO getSignatureDao() {
  private void createSchema() {
  createLibrariesTable();
  createSignaturesTable();
+ createOracleLibrariesTable();
+ createDependenciesTable();
+ createPluginsTable();
+ createPluginConfigTable();
 // createLibrarySignatureTable();
 // addIndexes();
  }
 
+ private void createDependenciesTable() {
+ String createTableQuery = "CREATE TABLE IF NOT EXISTS dependencies (" +
+ "id INT PRIMARY KEY AUTO_INCREMENT, " +
+ "group_id VARCHAR(255) NOT NULL, " +
+ "artifact_id VARCHAR(255) NOT NULL," +
+ "version VARCHAR(255) NOT NULL, " +
+ "scope VARCHAR(255))";
+
+ try (Connection connection = ds.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.executeUpdate(createTableQuery);
+ logger.info("Dependencies table created or already exists.");
+ } catch (SQLException e) {
+ logger.error("Error while creating dependencies table.", e);
+ }
+ }
+
+ private void createPluginsTable() {
+ String createTableQuery = "CREATE TABLE IF NOT EXISTS plugins (" +
+ "id INT PRIMARY KEY AUTO_INCREMENT, " +
+ "group_id VARCHAR(255), " +
+ "artifact_id VARCHAR(255) NOT NULL, " +
+ "version VARCHAR(255))";
+
+ try (Connection connection = ds.getConnection();
+Statement statement = connection.createStatement()) {
+ statement.executeUpdate(createTableQuery);
+ logger.info("Plugins table created or already exists.");
+ } catch (SQLException e) {
+ logger.error("Error while creating plugins table.", e);
+ }
+ }
+
+ private void createPluginConfigTable() {
+ String createTableQuery = "CREATE TABLE IF NOT EXISTS plugin_config (" +
+ "id INT PRIMARY KEY AUTO_INCREMENT, " +
+ "plugin_id INT NOT NULL, " +
+ "config_key VARCHAR(255) NOT NULL, " +
+ "config_value VARCHAR(1020))";
+
+ try (Connection connection = ds.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.executeUpdate(createTableQuery);
+ logger.info("Plugin config table created or already exists.");
+ } catch (SQLException e) {
+ logger.error("Error while creating plugin config table.", e);
+ }
+ }
+
  private void addIndexes() {
  String createLibraryIdIndexQuery = "CREATE INDEX idx_library_id ON library_signature (library_id)";
  String createSignatureIdIndexQuery = "CREATE INDEX idx_signature_id ON library_signature (signature_id)";
@@ -76,6 +129,22 @@ private void addIndexes() {
  }
  }
 
+ private void createOracleLibrariesTable() {
+ String createTableQuery = "CREATE TABLE IF NOT EXISTS oracle_libraries (" +
+ "id INT PRIMARY KEY AUTO_INCREMENT, "
+ + "group_id VARCHAR(255) NOT NULL, "
+ + "artifact_id VARCHAR(255) NOT NULL, "
+ + "version VARCHAR(255) NOT NULL)";
+
+ try (Connection connection = ds.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.executeUpdate(createTableQuery);
+ logger.info("Oracle libraries table created or already exists.");
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
  private void createLibrariesTable() {
  String createTableQuery = "CREATE TABLE IF NOT EXISTS libraries (" +
  "id INT PRIMARY KEY AUTO_INCREMENT, "

src/main/java/nl/tudelft/cornul11/thesis/corpus/database/SignatureDAO.java

@@ -3,12 +3,14 @@
 import nl.tudelft.cornul11.thesis.corpus.file.JarInfoExtractor;
 import nl.tudelft.cornul11.thesis.corpus.file.LibraryMatchInfo;
 import nl.tudelft.cornul11.thesis.corpus.model.Signature;
+import org.apache.maven.model.Model;
 
 import java.util.List;
 
 public interface SignatureDAO {
  int insertLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc, boolean isBrokenJar);
  int insertSignatures(List<Signature> signatures, long jarHash, long jarCrc);
+ void insertPluginInfo(Model model);
  List<LibraryMatchInfo> returnTopLibraryMatches(List<Long> hashes);
  void closeConnection();
 }

src/main/java/nl/tudelft/cornul11/thesis/corpus/database/SignatureDAOImpl.java

@@ -4,13 +4,16 @@
 import nl.tudelft.cornul11.thesis.corpus.file.JarInfoExtractor;
 import nl.tudelft.cornul11.thesis.corpus.file.LibraryMatchInfo;
 import nl.tudelft.cornul11.thesis.corpus.model.Signature;
+import org.apache.maven.model.Build;
+import org.apache.maven.model.Dependency;
+import org.apache.maven.model.Model;
+import org.apache.maven.model.Plugin;
+import org.codehaus.plexus.util.xml.Xpp3Dom;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.sql.*;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 
@@ -138,6 +141,83 @@ public List<LibraryMatchInfo> returnTopLibraryMatches(List<Long> hashes) {
  return libraryHashesCount;
  }
 
+ public void insertPluginInfo(Model model) {
+ long startTime = System.currentTimeMillis();
+
+ String insertLibraryQuery = "INSERT INTO oracle_libraries (group_id, artifact_id, version) VALUES (?, ?, ?)";
+ String insertDependencyQuery = "INSERT INTO dependencies (group_id, artifact_id, version, scope) VALUES (?, ?, ?, ?)";
+ String insertPluginQuery = "INSERT INTO plugins (group_id, artifact_id, version) VALUES (?, ?, ?)";
+ String insertPluginConfigQuery = "INSERT INTO plugin_config (plugin_id, config_key, config_value) VALUES (?, ?, ?)";
+
+ executeWithDeadlockRetry(connection -> {
+ try {
+ PreparedStatement libraryStatement = connection.prepareStatement(insertLibraryQuery, Statement.RETURN_GENERATED_KEYS);
+ libraryStatement.setString(1, model.getGroupId());
+ libraryStatement.setString(2, model.getArtifactId());
+ libraryStatement.setString(3, model.getVersion());
+ libraryStatement.executeUpdate();
+
+ List<Dependency> dependencies = model.getDependencies();
+ PreparedStatement dependencyStatement = connection.prepareStatement(insertDependencyQuery);
+ for (Dependency dependency : dependencies) {
+ dependencyStatement.setString(1, dependency.getGroupId());
+ dependencyStatement.setString(2, dependency.getArtifactId());
+ dependencyStatement.setString(3, dependency.getVersion());
+ dependencyStatement.setString(4, dependency.getScope());
+ dependencyStatement.executeUpdate();
+ }
+
+ Build build = model.getBuild();
+ if (build != null) {
+ List<Plugin> plugins = build.getPlugins();
+ for (Plugin plugin : plugins) {
+ PreparedStatement pluginStatement = connection.prepareStatement(insertPluginQuery, Statement.RETURN_GENERATED_KEYS);
+ pluginStatement.setString(1, plugin.getGroupId());
+ pluginStatement.setString(2, plugin.getArtifactId());
+ pluginStatement.setString(3, plugin.getVersion());
+ pluginStatement.executeUpdate();
+
+ ResultSet generatedKeys = pluginStatement.getGeneratedKeys();
+ if (generatedKeys.next()) {
+ int pluginId = generatedKeys.getInt(1);
+
+ Map<String, String> configValues = parsePluginConfig(plugin);
+ PreparedStatement configStatement = connection.prepareStatement(insertPluginConfigQuery);
+ for (Map.Entry<String, String> entry : configValues.entrySet()) {
+ configStatement.setInt(1, pluginId);
+ configStatement.setString(2, entry.getKey());
+ configStatement.setString(3, entry.getValue());
+ configStatement.executeUpdate();
+ }
+ }
+ }
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ });
+
+ long endTime = System.currentTimeMillis();
+ logger.info("Inserting plugin info took " + (endTime - startTime) / 1000.0 + " seconds.");
+
+ }
+
+ public Map<String, String> parsePluginConfig(Plugin plugin) {
+ Map<String, String> configValues = new HashMap<>();
+
+ Xpp3Dom config = (Xpp3Dom) plugin.getConfiguration();
+ if (config != null) {
+ for (Xpp3Dom child : config.getChildren()) {
+ String key = child.getName();
+ String value = child.getValue();
+ configValues.put(key, value);
+ }
+ }
+
+ return configValues;
+ }
+
  private void executeWithDeadlockRetry(Consumer<Connection> action) {
  boolean success = false;
  while (!success) {

src/main/java/nl/tudelft/cornul11/thesis/oracle/OracleMainApp.java

@@ -0,0 +1,90 @@
+package nl.tudelft.cornul11.thesis.oracle;
+
+import nl.tudelft.cornul11.thesis.corpus.database.DatabaseConfig;
+import nl.tudelft.cornul11.thesis.corpus.database.DatabaseManager;
+import nl.tudelft.cornul11.thesis.corpus.database.SignatureDAO;
+import nl.tudelft.cornul11.thesis.corpus.util.ConfigurationLoader;
+import org.apache.maven.model.Model;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class OracleMainApp {
+ private static final Logger logger = LoggerFactory.getLogger(OracleMainApp.class);
+ private final ExecutorService executor;
+ private final Map<String, Model> modelCache;
+
+ public OracleMainApp(int threadCount) {
+ this.executor = Executors.newFixedThreadPool(threadCount);
+ this.modelCache = new ConcurrentHashMap<>();
+ }
+
+ public void processPomFiles(Path pathToFile) {
+ ConfigurationLoader config = new ConfigurationLoader();
+ DatabaseConfig databaseConfig = config.getDatabaseConfig();
+ DatabaseManager databaseManager = DatabaseManager.getInstance(databaseConfig);
+ SignatureDAO signatureDao = databaseManager.getSignatureDao();
+
+ List<String> usingShadePlugin = new CopyOnWriteArrayList<>();
+ AtomicInteger brokenPomCount = new AtomicInteger(0);
+ AtomicInteger processedPomCount = new AtomicInteger(0);
+ try {
+ List<String> allLines;
+ long startTime = System.currentTimeMillis();
+ long totalLineCount;
+ logger.info("Processing files listed in: " + pathToFile);
+
+ try (Stream<String> lines = Files.lines(pathToFile)) {
+ allLines = lines.collect(Collectors.toList());
+ totalLineCount = allLines.size();
+ }
+
+ allLines.stream().map(Paths::get).forEach(path -> {
+ executor.submit(new PomProcessor(path, modelCache, usingShadePlugin, brokenPomCount, processedPomCount, signatureDao));
+ });
+
+ executor.shutdown();
+ while (!executor.isTerminated()) {
+ try {
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+ } catch (InterruptedException e) {
+ List<Runnable> remainingTasks = executor.shutdownNow(); // Force remaining tasks to terminate
+ logger.error("Interrupted while waiting for tasks to complete", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ long endTime = System.currentTimeMillis();
+ logger.info("Processed all POM files listed in " + pathToFile + " in " + (endTime - startTime) / 1000 + " seconds");
+
+ if (usingShadePlugin.isEmpty()) {
+ logger.info("No POM files using the shade plugin were found");
+ } else {
+ logger.info("POM files using the shade plugin:");
+// usingShadePlugin.forEach(logger::info);
+ logger.info("Total: " + usingShadePlugin.size());
+ // print percentage of total
+ logger.info("Percentage of total: " + (double) usingShadePlugin.size() / totalLineCount * 100 + "%");
+ }
+ logger.warn("Broken POM files: " + brokenPomCount.get());
+ } catch (IOException e) {
+ logger.error("Error while processing files", e);
+ }
+ }
+
+ public static void main(String[] args) {
+ Path pathToFile = Paths.get("paths/pom_files.txt");
+ OracleMainApp app = new OracleMainApp(1);
+ app.processPomFiles(pathToFile);
+ }
+}