implemented github API license URL extraction for remote github repos…

…itory links

core/src/main/java/com/devonfw/tools/solicitor/licensetexts/StrategyLicenseUrlGuesser.java

@@ -3,14 +3,20 @@
  */
 package com.devonfw.tools.solicitor.licensetexts;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;
 
+import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 
 import com.devonfw.tools.solicitor.SolicitorVersion;
+import com.devonfw.tools.solicitor.common.SolicitorRuntimeException;
 import com.devonfw.tools.solicitor.common.content.ContentProvider;
 import com.devonfw.tools.solicitor.common.content.web.WebContent;
 
@@ -24,6 +30,9 @@ public class StrategyLicenseUrlGuesser implements LicenseUrlGuesser {
   private ContentProvider<WebContent> webContentProvider;
 
   private SolicitorVersion solicitorVersion;
+
+  @Value("${solicitor.githubtoken}")
+  private String token;
 
   /**
    * The constructor.
@@ -56,14 +65,19 @@ private void setTrace(String trace, StringBuilder traceBuilder) {
     traceBuilder.append(trace).append('\n');
   }
 
-  // https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.stat
-  // https://raw.githubusercontent.com/nodelib/nodelib/master/packages/fs/fs.stat/README.md
   // helper method that normalizes a github url and retrieves the raw link to
   // a given license
   private String normalizeGitURL(String url, StringBuilder traceBuilder) {
-
     String oldURL = url;
-    if (url.contains("github")) {
+    // case that github remote repository link is given
+    if (url.contains("github.com") && url.endsWith(".git")) {
+      url = githubAPILicenseUrl(url, token);
+      if(!url.equals(oldURL) && !url.contains("api.github.com")) {
+        setTrace("URL changed from " + oldURL + " to " + url, traceBuilder);
+        return url;
+      }
+    }
+    if (url.contains("github.com")) {
       // use https for all github URLs
       url = url.replace("http:", "https:");
       // omit repo suffix if existent
@@ -154,4 +168,44 @@ public GuessedLicenseUrlContent getContentForUri(String uri) {
     return new GuessedLicenseUrlContent(guessedUrl, auditLogBuilder.toString());
   }
 
+  //tries to get github license file location based of vsc-link 
+  public String githubAPILicenseUrl(String link, String token) {
+
+    String result = "";
+    if (link.contains("github.com")) {
+      if (link.endsWith(".git")) {
+        link = link.substring(0, link.length() - 4);
+    }
+    link = link.replace("git://", "https://");
+    link = link.replace("ssh://", "https://");
+    link = link.replace("git@", "");
+    if (!link.contains("api.github.com")) {
+      link = link.replace("github.com/", "api.github.com/repos/");
+      link = link.concat("/license");
+    }
+
+    String command = "curl -H \"Accept: application/vnd.github+json\" -H \"Authorization: token "+ token + "\" -i " + link;
+    ProcessBuilder processBuilder = new ProcessBuilder(command.split(" "));
+    try {
+      Process process = processBuilder.start();
+      InputStream inputStream = process.getInputStream();
+      result = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+        if (result.contains("download_url")) {
+          result = result.substring(result.indexOf("\"download_url\": "));
+          result = result.substring(17,result.indexOf(",")-1);
+        }
+        if (result.contains("\"message\": \"Moved Permanently\"")) {
+          String tempLink = result.substring(result.indexOf("\"url\": "));
+          tempLink = tempLink.substring(17,result.indexOf(",")-1);
+          result = githubAPILicenseUrl(tempLink, token);
+        }
+        if (result.contains("\"message\": \"Not Found\"")) {
+           result = link;
+         }
+      } catch (IOException e) {
+        throw new SolicitorRuntimeException("Could not handle command call for api request'" + command + "'", e);
+      }
+    }
+    return result;
+  }  
 }

core/src/main/java/com/devonfw/tools/solicitor/reader/ort/OrtReader.java

@@ -61,12 +61,12 @@ public void readInventory(String type, String sourceUrl, Application application
     	Map singlePackage = (Map) iterator.get("package");
 		String id = (String) singlePackage.get("id");
 		Map vcsProcessed = (Map) singlePackage.get("vcs_processed");
-		String repo = (String) vcsProcessed.get("url");
+		String licenseUrl = (String) vcsProcessed.get("url");
 		String pURL = (String) singlePackage.get("purl");
 
 		String homePage = (String) singlePackage.get("homepage_url");
 		if (homePage == null || homePage.isEmpty()) {
-		  homePage = repo;
+		  homePage = licenseUrl;
 		}
 
         ApplicationComponent appComponent = getModelFactory().newApplicationComponent();
@@ -92,11 +92,11 @@ public void readInventory(String type, String sourceUrl, Application application
         List lic = (List) singlePackage.get("declared_licenses");
         if (lic.isEmpty()) {
           // add empty raw license if no license info attached
-          addRawLicense(appComponent, null, null, sourceUrl);
+          addRawLicense(appComponent, null, licenseUrl, sourceUrl);
         } else {
           for (Object cl : lic) {
             licenseCount++;
-            addRawLicense(appComponent, cl.toString(), null, sourceUrl);
+            addRawLicense(appComponent, cl.toString(), licenseUrl, sourceUrl);
           }
         }     
       doLogging(sourceUrl, application, componentCount, licenseCount);

core/src/main/resources/application.properties

@@ -41,6 +41,9 @@ solicitor.classpath-guessedlicenseurl-cache-locations=licenseurls
 # Deprecated features are deactivated by default. If set to true they might be (temporarily) activated.
 solicitor.deprecated-features-allowed=false
 
+# input for personal github api token to increase ORT-reader rate limit
+solicitor.githubtoken=
+
 ## Feature flags for activation of non-standard/experimental functionality
 # Incorporate scancode infos into model
 solicitor.feature-flag.scancode=false

core/src/test/resources/analyzer-result.json

@@ -49,7 +49,7 @@
         },
         "vcs_processed" : {
           "type" : "",
-          "url" : "",
+          "url" : "ssh://[email protected]/hamcrest/JavaHamcrest.git",
           "revision" : "",
           "path" : ""
         },
@@ -92,7 +92,7 @@
           },
           "vcs_processed" : {
             "type" : "Git",
-            "url" : "https://github.com/testproject.git",
+            "url" : "ssh://git@github.com/hamcrest/JavaHamcrest.git",
             "revision" : "",
             "path" : "testproject"
           }