performance improvement for timestamp lookup, better hostname lookup

src/main/java/nl/tudelft/cornul11/thesis/packaging/ProjectGenerator.java

@@ -5,6 +5,7 @@
 import nl.tudelft.cornul11.thesis.corpus.jarfile.JarProcessingUtils;
 import nl.tudelft.cornul11.thesis.corpus.model.Dependency;
 import nl.tudelft.cornul11.thesis.corpus.model.LibraryInfo;
+import nl.tudelft.cornul11.thesis.packaging.util.HostNameUtility;
 import org.apache.maven.shared.invoker.*;
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.Opcodes;
@@ -14,6 +15,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.net.InetAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -179,18 +181,16 @@ private boolean hasPublicDefaultConstructor(ClassNode classNode) {
 
  private String getJarLocation(Dependency library) {
  Path m2RepositoryPath;
+ // TODO: find a better way to determine the m2 repository path
  // get hostname to determine if we are on the server or not
- String hostname = System.getenv("HOSTNAME");
+ String hostname = HostNameUtility.getHostName();
 
  // sometimes hostname is null
- if (hostname == null) {
- hostname = "";
- }
+ hostname = Objects.requireNonNullElse(hostname, "");
 
  if (hostname.equals("goteborg")) {
- // m2RepositoryPath shoud become /data/.m2/repository
+ // m2RepositoryPath should become /data/.m2/repository
  m2RepositoryPath = Paths.get("/data", ".m2", "repository");
-
  } else {
  String userHomeDir = System.getProperty("user.home");
  m2RepositoryPath = Paths.get(userHomeDir, ".m2", "repository");

src/main/java/nl/tudelft/cornul11/thesis/packaging/util/HostNameUtility.java

@@ -0,0 +1,13 @@
+package nl.tudelft.cornul11.thesis.packaging.util;
+
+import java.net.InetAddress;
+
+public class HostNameUtility {
+ public static String getHostName() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ } catch (Exception e) {
+ return "";
+ }
+ }
+}

util/api_benchmark.py

@@ -0,0 +1,86 @@
+import json
+import statistics
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pprint import pprint
+
+import requests
+
+
+def analyze_results(results, max_time):
+ assert len(results) > 0
+ response_times = [result[1] for result in results if result[0] == 200]
+ error_count = len([result for result in results if result[0] != 200])
+ assert len(response_times) > 0
+ assert len(results) == len(response_times) + error_count
+ average_time = statistics.mean(response_times)
+ median_time = statistics.median(response_times)
+ min_time = min(response_times)
+ max_response_time = max(response_times)
+ std_dev = statistics.stdev(response_times)
+ percentiles = {
+ "50th": statistics.quantiles(response_times, n=100)[49],
+ "90th": statistics.quantiles(response_times, n=100)[89],
+ "95th": statistics.quantiles(response_times, n=100)[94],
+ "99th": statistics.quantiles(response_times, n=100)[98],
+ }
+ error_rate = error_count / len(results)
+ throughput = len(results) / max_time
+
+ return {
+ "average": average_time,
+ "median": median_time,
+ "min": min_time,
+ "max_response_time": max_response_time,
+ "std_dev": std_dev,
+ "percentiles": percentiles,
+ "error_rate": error_rate,
+ "throughput": throughput,
+ }
+
+
+def parse_jar_path(jar_path):
+ parts = jar_path.strip().split("/")
+ version = parts[-2]
+ artifactId = parts[-3]
+ groupId = ".".join(parts[parts.index("repository") + 1 : -3])
+ return groupId, artifactId, version
+
+
+def benchmark_api(groupId, artifactId, version, url="http://localhost:8080/lookup"):
+ params = {
+ "groupId": groupId,
+ "artifactId": artifactId,
+ "version": version,
+ }
+ start = time.time()
+ response = requests.get(url, params=params)
+ end = time.time()
+ return response.status_code, end - start
+
+
+def main(file_path):
+ with open(file_path, "r") as file:
+ jar_paths = file.readlines()
+
+ start_time = time.time()
+
+ with ThreadPoolExecutor(max_workers=10) as executor:
+ futures = [
+ executor.submit(benchmark_api, *parse_jar_path(path)) for path in jar_paths
+ ]
+ results = [future.result() for future in futures]
+
+ end_time = time.time()
+ max_time = end_time - start_time
+
+ analysis = analyze_results(results, max_time)
+
+ pprint(analysis)
+
+
+if __name__ == "__main__":
+ import sys
+
+ file_path = sys.argv[1]
+ main(file_path)

util/maven-index-timestamp-lookup/src/main/java/org/cornul11/maven/MavenIndexTimestampLookup.java

@@ -49,9 +49,6 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 import static java.util.Objects.requireNonNull;
 import static spark.Spark.get;
@@ -69,14 +66,15 @@ public class MavenIndexTimestampLookup {
 
  private final Map<String, IndexCreator> indexCreators;
 
- private final ExecutorService executorService = Executors.newCachedThreadPool();
-
  public static void main(String[] args) throws IOException {
  final com.google.inject.Module app = Main.wire(BeanScanning.INDEX);
  MavenIndexTimestampLookup mavenIndexTimestampLookup = Guice.createInjector(app).getInstance(MavenIndexTimestampLookup.class);
 
  if (args.length > 0 && "server".equals(args[0])) {
- mavenIndexTimestampLookup.startServer();
+ mavenIndexTimestampLookup.updateLocalIndex();
+
+ IndexingContext centralIndex = mavenIndexTimestampLookup.createCentralContext();
+ mavenIndexTimestampLookup.startServer(centralIndex);
  } else {
  mavenIndexTimestampLookup.perform(args);
  }
@@ -113,24 +111,19 @@ private IndexingContext createCentralContext() throws IOException {
  indexers);
  }
 
- private String lookupArtifactLastModified(String groupId, String artifactId, String version) throws IOException {
- IndexingContext centralContext = createCentralContext();
- try {
- // construct the query for known GA
- Query query = constructQuery(groupId, artifactId, version);
+ private String lookupArtifactLastModified(IndexingContext centralContext, String groupId, String artifactId, String version) throws IOException {
+ // construct the query for known GA
+ Query query = constructQuery(groupId, artifactId, version);
 
-  IteratorSearchRequest request = new IteratorSearchRequest(query, Collections.singletonList(centralContext));
-  IteratorSearchResponse response = indexer.searchIterator(request);
+ IteratorSearchRequest request = new IteratorSearchRequest(query, Collections.singletonList(centralContext));
+ IteratorSearchResponse response = indexer.searchIterator(request);
 
- if (response.getTotalHitsCount() > 0) {
- ArtifactInfo ai = response.iterator().next();
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
- return sdf.format(new Date(ai.getLastModified()));
- } else {
- return "Artifact not found";
- }
- } finally {
- indexer.closeIndexingContext(centralContext, false);
+ if (response.getTotalHitsCount() > 0) {
+ ArtifactInfo ai = response.iterator().next();
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+ return sdf.format(new Date(ai.getLastModified()));
+ } else {
+ return "Artifact not found";
  }
  }
 
@@ -156,10 +149,6 @@ private Query constructQuery(String groupId, String artifactId, String version)
  public void perform(String[] args) throws IOException {
  IndexingContext centralContext = createCentralContext();
  try {
- if (args[0].equals("update")) {
- updateLocalIndex(centralContext);
- return;
- }
  String groupId = args[0];
  String artifactId = args[1];
  String version = args[2];
@@ -182,7 +171,7 @@ private void executeQuery(IndexingContext context, Query query) throws IOExcepti
  }
  }
 
- private void startServer() {
+ private void startServer(IndexingContext centralIndex) throws IOException {
  port(8080);
 
  get("/lookup", (request, response) -> {
@@ -195,30 +184,38 @@ private void startServer() {
  return "Missing query parameters";
  }
 
- CompletableFuture<String> future = CompletableFuture.supplyAsync(() -> {
- try {
- String lastModified = lookupArtifactLastModified(groupId, artifactId, version);
- if (!"Artifact not found".equals(lastModified)) {
- return lastModified;
- } else {
- response.status(404);
- return "Artifact not found";
- }
- } catch (Exception e) {
- response.status(500);
- return "Internal server error: " + e.getMessage();
+ try {
+ String lastModified = lookupArtifactLastModified(centralIndex, groupId, artifactId, version);
+ if (!"Artifact not found".equals(lastModified)) {
+ return lastModified;
+ } else {
+ response.status(404);
+ return "Artifact not found";
  }
- }, executorService);
-
- return future.join();
+ } catch (Exception e) {
+ response.status(500);
+ e.printStackTrace();
+ return "Internal server error: " + e.getMessage();
+ }
  });
+
+ // when the server is stopped, close the indexer
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ indexer.closeIndexingContext(centralIndex, false);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }));
  }
 
- private void updateLocalIndex(IndexingContext centralContext) throws IOException {
+ private void updateLocalIndex() throws IOException {
  Instant updateStart = Instant.now();
- System.out.println("Updating Index...");
+ System.out.println("Updating index...");
  System.out.println("This might take a while on first run, so please be patient!");
 
+ IndexingContext centralContext = createCentralContext();
+
  Date centralContextCurrentTimestamp = centralContext.getTimestamp();
  IndexUpdateRequest updateRequest = new IndexUpdateRequest(centralContext, new Java11HttpClient());
  IndexUpdateResult updateResult = indexUpdater.fetchAndUpdateIndex(updateRequest);
@@ -231,6 +228,7 @@ private void updateLocalIndex(IndexingContext centralContext) throws IOException
  + " - " + updateResult.getTimestamp() + " period.");
  }
 
+ indexer.closeIndexingContext(centralContext, false);
  System.out.println("Finished in "
  + Duration.between(updateStart, Instant.now()).getSeconds() + " sec");
  System.out.println();

util/pom_analysis.py

@@ -13,8 +13,6 @@
 from tqdm.contrib.logging import logging_redirect_tqdm
 from tqdm.gui import tqdm as tqdm_gui
 
-total_waiting_for_maven = 0
-
 DEFAULT_ARCHIVE_PATH = "pom_archive.zip"
 NS_URL = "http://maven.apache.org/POM/4.0.0"
 
@@ -288,12 +286,13 @@ def print_stats(self):
  logging.info(
  f"Total pom files with parent: {self.total_with_parents} ({self.total_with_parents / self.total_pom_files * 100:.2f}%)"
  )
- logging.info(
- f"Total not found in index: {self.total_not_found_in_index} ({self.total_not_found_in_index / self.total_shade_plugins * 100:.2f}%)"
- )
- logging.info(
- f"Total not found: {self.total_not_found} ({self.total_not_found / self.total_shade_plugins * 100:.2f}%)"
- )
+ if self.total_shade_plugins > 0:
+ logging.info(
+ f"Total not found in index: {self.total_not_found_in_index} ({self.total_not_found_in_index / self.total_shade_plugins * 100:.2f}%)"
+ )
+ logging.info(
+ f"Total not found: {self.total_not_found} ({self.total_not_found / self.total_shade_plugins * 100:.2f}%)"
+ )
 
  def save_stats_if_required(self):
  if self.args.save:
@@ -313,11 +312,8 @@ def save_stats_if_required(self):
  }
 
  with open(self.args.save, "w") as f:
- json.dump(stats, f)
+ json.dump(stats, f, indent=4)
  logging.info(f"Saved stats to {self.args.save}")
- logging.info(
- f"Total time waiting for Maven Central: {total_waiting_for_maven}"
- )
  logging.info(
  f"Total time: {(datetime.now() - self.start_time).total_seconds()}"
  )
@@ -333,7 +329,11 @@ def parse_arguments():
  default=None,
  )
  parser.add_argument(
- "--file", type=str, help="File containing paths to pom.xml files", default=None
+ "-f",
+ "--file",
+ type=str,
+ help="File containing paths to pom.xml files",
+ default=None,
  )
  parser.add_argument(
  "--mode",
@@ -373,9 +373,6 @@ def get_publication_date_from_maven_repo_header(group_id, artifact_id, version):
 
 
 def get_publication_date_from_maven_repo(group_id, artifact_id, version):
- global total_waiting_for_maven
- start_time = datetime.now()
-
  group_id = group_id.replace(".", "/")
 
  url = f"https://repo1.maven.org/maven2/{group_id}/{artifact_id}/{version}"
@@ -406,14 +403,9 @@ def get_publication_date_from_maven_repo(group_id, artifact_id, version):
  except Exception as err:
  logging.error(f"Other error occurred: {err}")
  return None
- finally:
- end_time = datetime.now()
- total_waiting_for_maven += (end_time - start_time).total_seconds()
 
 
 def get_publication_date_from_maven_central(group_id, artifact_id, version):
- global total_waiting_for_maven
- start_time = datetime.now()
  try:
  base_url = "https://search.maven.org/solrsearch/select"
  query = f"g:{group_id} AND a:{artifact_id} AND v:{version}"
@@ -436,9 +428,6 @@ def get_publication_date_from_maven_central(group_id, artifact_id, version):
  f"Error fetching publication date for {group_id}:{artifact_id}:{version} from Maven Central: {e}"
  )
  return None
- finally:
- end_time = datetime.now()
- total_waiting_for_maven += (end_time - start_time).total_seconds()
 
 
 def main():