improved the signature collection

src/main/java/nl/tudelft/cornul11/thesis/MainApp.java

@@ -1,16 +1,13 @@
 package nl.tudelft.cornul11.thesis;
 
 import nl.tudelft.cornul11.thesis.api.PostRequestClient;
-import nl.tudelft.cornul11.thesis.commandline.OptionsBuilder;
 import nl.tudelft.cornul11.thesis.commandline.CommandExecutor;
-
+import nl.tudelft.cornul11.thesis.commandline.OptionsBuilder;
 import nl.tudelft.cornul11.thesis.util.ConfigurationLoader;
-import org.apache.commons.cli.*;
+import org.apache.commons.cli.ParseException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Properties;
-
 public class MainApp {
     private static final Logger logger = LoggerFactory.getLogger(MainApp.class);
 

src/main/java/nl/tudelft/cornul11/thesis/database/DatabaseManager.java

@@ -54,7 +54,7 @@ private void createSchema() {
         createLibrariesTable();
         createSignaturesTable();
         createLibrarySignatureTable();
-        addIndexes();
+//        addIndexes();
     }
 
     private void addIndexes() {

src/main/java/nl/tudelft/cornul11/thesis/database/SignatureDAOImpl.java

@@ -173,8 +173,8 @@ private void executeWithDeadlockRetry(Consumer<Connection> action) {
     private void handleDeadlock() {
         logger.error("Deadlock detected. Retrying...");
         try {
-            // sleep for a random amount of time between 1 and 2 seconds
-            Thread.sleep(1000 + (int) (Math.random() * 1000));
+            // sleep for a random amount of time between 0.5 and 1 seconds
+            Thread.sleep(500 + (int) (Math.random() * 500));
         } catch (InterruptedException ie) {
             Thread.currentThread().interrupt();
             throw new RuntimeException(ie);

src/main/java/nl/tudelft/cornul11/thesis/jarfile/FileAnalyzer.java

@@ -70,22 +70,22 @@ public void printIgnoredUberJars() {
 
     public int processJarFile(Path jarFilePath) {
         JarHandler jarHandler = new JarHandler(jarFilePath, ignoredUberJars, insertedLibraries, config);
-        List<ClassFileInfo> classFileInfos = jarHandler.extractJarFileInfo();
+        List<ClassFileInfo> signatures = jarHandler.extractSignatures();
 
         StringBuilder sb = new StringBuilder();
-        for (ClassFileInfo classFileInfo : classFileInfos) {
-            sb.append(classFileInfo.getHashCode());
+        for (ClassFileInfo signature : signatures) {
+            sb.append(signature.getHashCode());
         }
         LongHashFunction xx = LongHashFunction.xx();
         long jarHash = xx.hashChars(sb.toString());
         long jarCrc = jarHandler.getCrc();
 
         JarInfoExtractor jarInfoExtractor = new JarInfoExtractor(jarFilePath.toString());
-        if (classFileInfos.isEmpty()) { // it's probably an uber-JAR, let's still add it to the db
+        if (signatures.isEmpty()) { // it's probably an uber-JAR, let's still add it to the db
             return commitLibrary(jarInfoExtractor, jarHash, jarCrc);
         }
 
-        return commitSignatures(classFileInfos, jarInfoExtractor, jarHash, jarCrc);
+        return commitSignatures(signatures, jarInfoExtractor, jarHash, jarCrc);
     }
 
     public int commitLibrary(JarInfoExtractor jarInfoExtractor, long jarHash, long jarCrc) {

src/main/java/nl/tudelft/cornul11/thesis/jarfile/JarHandler.java

@@ -56,13 +56,12 @@ private long generateCrc() {
         return crc.getValue();
     }
 
-    public List<ClassFileInfo> extractJarFileInfo() {
+    public List<ClassFileInfo> extractSignatures() {
         mavenSubmodules.clear();
 
         logger.info("Attempting to process " + jarFilePath);
         try (JarFile jarFile = new JarFile(jarFilePath.toFile())) {
             Enumeration<JarEntry> entries = jarFile.entries();
-            String initialClassPrefix = null;
             List<ClassFileInfo> classFileInfos = new ArrayList<>();
             logger.info("Processing " + jarFilePath + " with " + jarFile.size() + " entries");
             while (entries.hasMoreElements()) {
@@ -171,7 +170,6 @@ private boolean hasMultiplePackages(Path jarFilePath, JarEntry entry, String ini
     }
 
     private ClassFileInfo processClassFile(JarEntry entry, JarFile jarFile) throws IOException {
-//        logger.debug("Processing class file: " + entry.getName()); // TODO: this makes the logs too verbose
         try (InputStream classFileInputStream = jarFile.getInputStream(entry)) {
             byte[] bytecode = classFileInputStream.readAllBytes();
             BytecodeDetails bytecodeDetails = BytecodeParser.extractSignature(bytecode);

src/main/java/nl/tudelft/cornul11/thesis/jarfile/JarSignatureMapper.java

@@ -20,7 +20,7 @@ public class JarSignatureMapper {
     private int totalClassCount = 0;
     private final SignatureDAO signatureDao;
     private final Logger logger = LoggerFactory.getLogger(JarSignatureMapper.class);
-
+    private static final Set<String> FILENAME_EXCEPTIONS = Set.of("module-info.class", "package-info.class");
     public JarSignatureMapper(SignatureDAO signatureDao) {
         this.signatureDao = signatureDao;
     }
@@ -33,7 +33,7 @@ public Map<String, Map<String, Object>> inferJarFile(Path jarFilePath) {
             while (entries.hasMoreElements()) {
                 JarEntry entry = entries.nextElement();
 
-                if (!entry.isDirectory() && entry.getName().endsWith(".class")) {
+                if (!entry.isDirectory() && entry.getName().endsWith(".class") && !FILENAME_EXCEPTIONS.contains(entry.getName())) {
                     classFileCount++;
                     ClassFileInfo classFileInfo = processClassFile(entry, jarFile);
                     if (classFileInfo != null) {

src/main/java/nl/tudelft/cornul11/thesis/signature/extractor/bytecode/BytecodeAnnotationVisitor.java

@@ -16,12 +16,16 @@ public BytecodeAnnotationVisitor(int api, AnnotationVisitor annotationVisitor, A
 
     @Override
     public void visit(String name, Object value) {
+        if (value instanceof String) {
+            value = BytecodeUtils.getShortName((String) value);
+        }
         annotation.arguments.put(name, value);
         super.visit(name, value);
     }
 
     @Override
     public AnnotationVisitor visitArray(String name) {
+        name = BytecodeUtils.getShortName(name);
         List<Object> values = new ArrayList<>();
         annotation.arrayArguments.put(name, values);
         return new BytecodeArrayAnnotationVisitor(api, super.visitArray(name), values);
@@ -30,15 +34,16 @@ public AnnotationVisitor visitArray(String name) {
     @Override
     public AnnotationVisitor visitAnnotation(String name, String desc) {
         AnnotationDetails nestedAnnotation = new AnnotationDetails();
-        nestedAnnotation.desc = desc;
+        nestedAnnotation.desc = BytecodeUtils.getShortDesc(desc);
         annotation.annotationArguments.put(name, nestedAnnotation);
         return new BytecodeAnnotationVisitor(api, super.visitAnnotation(name, desc), nestedAnnotation);
     }
 
     @Override
     public void visitEnum(String name, String desc, String value) {
+        desc = BytecodeUtils.getShortDesc(desc);
+        value = BytecodeUtils.getShortName(value);
         annotation.arguments.put(name, desc + "." + value);
         super.visitEnum(name, desc, value);
     }
-    // Override other visit* methods as needed
 }

src/main/java/nl/tudelft/cornul11/thesis/signature/extractor/bytecode/BytecodeArrayAnnotationVisitor.java

@@ -14,12 +14,16 @@ public BytecodeArrayAnnotationVisitor(int api, AnnotationVisitor annotationVisit
 
     @Override
     public void visit(String name, Object value) {
+        if (value instanceof String) {
+            value = BytecodeUtils.getShortName((String) value);
+        }
         values.add(value);
         super.visit(name, value);
     }
 
     @Override
     public void visitEnum(String name, String desc, String value) {
+        desc = BytecodeUtils.getShortDesc(desc);
         values.add(desc + "." + value);
         super.visitEnum(name, desc, value);
     }

src/main/java/nl/tudelft/cornul11/thesis/signature/extractor/bytecode/BytecodeClassVisitor.java

@@ -4,6 +4,7 @@
 import org.objectweb.asm.*;
 
 import java.util.Arrays;
+import java.util.stream.Collectors;
 
 import static org.objectweb.asm.Opcodes.ASM9;
 
@@ -19,21 +20,15 @@ public BytecodeDetails getBytecodeClass() {
     }
 
     public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
-        bytecodeDetails.name = name;
-        bytecodeDetails.extendsType = superName;
-        bytecodeDetails.interfaces.addAll(Arrays.asList(interfaces));
+        bytecodeDetails.name = BytecodeUtils.getShortName(name);
+        bytecodeDetails.extendsType = BytecodeUtils.getShortName(superName);
+        bytecodeDetails.interfaces = Arrays.stream(interfaces).map(BytecodeUtils::getShortName).collect(Collectors.toList());
         super.visit(version, access, name, signature, superName, interfaces);
     }
 
-    public void visitSource(String source, String debug) {
-        // TODO:
-    }
-
-    public void visitOuterClass(String owner, String name, String desc) {
-        // TODO:
-    }
-
     public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+        desc = BytecodeUtils.getShortDesc(desc);
+
         AnnotationDetails annotationDetails = new AnnotationDetails();
         annotationDetails.desc = desc;
         annotationDetails.visible = visible;
@@ -43,55 +38,60 @@ public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
         return new BytecodeAnnotationVisitor(ASM9, av, annotationDetails);
     }
 
-    public void visitAttribute(Attribute attr) {
-        // TODO:
-    }
-
     public void visitInnerClass(String name, String outerName, String innerName, int access) {
-        // check if the inner class is an interface, if yes
-        // add it to the list of inner interfaces
+        name = BytecodeUtils.getShortName(name);
+        if (outerName != null)
+            outerName = BytecodeUtils.getShortName(outerName);
+
+        NestedClassDetails nestedClassDetails = new NestedClassDetails();
+        nestedClassDetails.name = name;
+        nestedClassDetails.outerName = outerName;
+        nestedClassDetails.innerName = innerName;
+        nestedClassDetails.access = access;
+
         if ((access & Opcodes.ACC_INTERFACE) != 0) {
-            InterfaceDetails interfaceDetails = new InterfaceDetails();
-            interfaceDetails.name = name;
-            interfaceDetails.outerName = outerName;
-            interfaceDetails.innerName = innerName;
-            interfaceDetails.access = access;
-            bytecodeDetails.innerInterfaces.add(interfaceDetails);
+            nestedClassDetails.type = "INTERFACE";
         } else if ((access & Opcodes.ACC_ENUM) != 0) {
-            // shouldn't forget that all inner classes and interfaces are compiled to separate .class files
-            // maybe the hash of a class should be the hash of all its inner classes and interfaces?
-            // TODO: consider looking at the class file name, and to group by everything up to $ in the name of the class
-            EnumDetails enumDetails = new EnumDetails();
-            enumDetails.name = name;
-            enumDetails.outerName = outerName;
-            enumDetails.innerName = innerName;
-            enumDetails.access = access;
-            bytecodeDetails.innerEnums.add(enumDetails);
+            nestedClassDetails.type = "ENUM";
+        } else if ((access & Opcodes.ACC_STATIC) != 0) {
+            nestedClassDetails.type = "STATIC_CLASS";
+        } else {
+            nestedClassDetails.type = "INNER_CLASS";
         }
+
+        bytecodeDetails.innerClasses.add(nestedClassDetails);
     }
 
     public FieldVisitor visitField(int access, String name, String desc, String signature, Object value) {
+        desc = BytecodeUtils.getShortDesc(desc);
+
         FieldDetails fieldDetails = new FieldDetails();
         fieldDetails.name = name;
         fieldDetails.desc = desc;
-        bytecodeDetails.fieldDetails.add(fieldDetails);
-        return super.visitField(access, name, desc, signature, value);
+        bytecodeDetails.fields.add(fieldDetails);
+
+        FieldVisitor originalVisitor = super.visitField(access, name, desc, signature, value);
+        return new BytecodeFieldVisitor(api, originalVisitor, fieldDetails);
     }
 
     public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) {
+        desc = BytecodeUtils.getShortDesc(desc);
+        signature = signature != null ? BytecodeUtils.getShortDesc(signature) : null;
+
         MethodDetails method = new MethodDetails();
+        method.access = access;
         method.name = name;
         method.setDesc(desc);
         method.signature = signature;
-        method.exceptions = exceptions;
+        method.exceptions = exceptions != null ? Arrays.stream(exceptions).map(BytecodeUtils::getShortName).toArray(String[]::new) : null;
 
         if ("<init>".equals(name)) {
             // This is a constructor
             ConstructorDetails constructor = new ConstructorDetails();
             constructor.name = name;
             constructor.desc = desc;
             constructor.signature = signature;
-            constructor.exceptions = exceptions;
+            constructor.exceptions = exceptions != null ? Arrays.stream(exceptions).map(BytecodeUtils::getShortName).toArray(String[]::new) : null;
             bytecodeDetails.constructors.add(constructor);
         } else {
             // This is a method
@@ -101,8 +101,4 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si
         MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions);
         return new BytecodeMethodVisitor(ASM9, mv, method);
     }
-
-    public void visitEnd() {
-        // nothing needs to be done apparently
-    }
 }

src/main/java/nl/tudelft/cornul11/thesis/signature/extractor/bytecode/BytecodeDetails.java

@@ -1,37 +1,47 @@
 package nl.tudelft.cornul11.thesis.signature.extractor.bytecode;
 
+import net.openhft.hashing.LongHashFunction;
 import nl.tudelft.cornul11.thesis.signature.extractor.bytecode.members.*;
 
 import java.util.ArrayList;
 import java.util.List;
 
-import net.openhft.hashing.LongHashFunction;
-
 public class BytecodeDetails {
     public String name;
     public String extendsType;
     public List<String> interfaces = new ArrayList<>();
-    public List<FieldDetails> fieldDetails = new ArrayList<>();
+    public List<FieldDetails> fields = new ArrayList<>();
     public List<MethodDetails> methods = new ArrayList<>();
     public List<ConstructorDetails> constructors = new ArrayList<>();
-    public List<InterfaceDetails> innerInterfaces = new ArrayList<>();
-    public List<EnumDetails> innerEnums = new ArrayList<>();
+    public List<NestedClassDetails> innerClasses = new ArrayList<>();
     public List<AnnotationDetails> annotations = new ArrayList<>();
 
+
+    //
+//    private void preProcessNames() {
+//        for (AnnotationDetails annotation : annotations) {
+//            for (Map.Entry<String, AnnotationDetails> entry : annotation.annotationArguments.entrySet()) {
+//                entry.getValue().desc = getShortDesc(entry.getValue().desc);
+//            }
+//        }
+//
+//        // Similar processing should be done for the names inside innerInterfaces, innerEnums, and annotations
+//    }
     public long getSignature() {
         // TODO: move the hashing function outside of this class
         // this function should only return the signature, not hash it
 
         LongHashFunction cityHashFunction = LongHashFunction.xx3();
         StringBuilder classSignature = new StringBuilder();
 
+        // Considering the name is now shortened (loses package name), we can add it to the signature
+        classSignature.append(name);
         classSignature.append(extendsType);
         classSignature.append(interfaces.toString());
-        classSignature.append(fieldDetails.toString());
+        classSignature.append(fields.toString());
         classSignature.append(methods.toString());
         classSignature.append(constructors.toString());
-        classSignature.append(innerInterfaces.toString());
-        classSignature.append(innerEnums.toString());
+        classSignature.append(innerClasses.toString());
         classSignature.append(annotations.toString());
         return cityHashFunction.hashChars(classSignature);
     }

src/main/java/nl/tudelft/cornul11/thesis/signature/extractor/bytecode/BytecodeFieldVisitor.java

@@ -0,0 +1,24 @@
+package nl.tudelft.cornul11.thesis.signature.extractor.bytecode;
+
+import nl.tudelft.cornul11.thesis.signature.extractor.bytecode.members.AnnotationDetails;
+import nl.tudelft.cornul11.thesis.signature.extractor.bytecode.members.FieldDetails;
+import org.objectweb.asm.AnnotationVisitor;
+import org.objectweb.asm.FieldVisitor;
+
+public class BytecodeFieldVisitor extends FieldVisitor {
+    private final FieldDetails field;
+
+    public BytecodeFieldVisitor(int api, FieldVisitor fieldVisitor, FieldDetails field) {
+        super(api, fieldVisitor);
+        this.field = field;
+    }
+
+    @Override
+    public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+        desc = BytecodeUtils.getShortDesc(desc);
+        AnnotationDetails annotation = new AnnotationDetails();
+        annotation.desc = desc;
+        field.annotations.add(annotation);
+        return new BytecodeAnnotationVisitor(api, super.visitAnnotation(desc, visible), annotation);
+    }
+}