Fix direct download https compressed qcow2 template checker (#7932)

This PR fixes an issue on direct download while registering HTTPS compressed files
Fixes: #7929

Author: nvazquez

core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java

@@ -183,7 +183,8 @@ public Long getRemoteFileSize(String url, String format) {
                 SSLContext context = getSSLContext();
                 urlConnection.setSSLSocketFactory(context.getSocketFactory());
                 urlConnection.connect();
-                return QCOW2Utils.getVirtualSize(urlConnection.getInputStream());
+                boolean isCompressed = !url.endsWith("qcow2");
+                return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed);
             } catch (IOException e) {
                 throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
             }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java

@@ -40,6 +40,11 @@ public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serv
         boolean checkResult = DirectDownloadHelper.checkUrlExistence(url);
         if (checkResult) {
             remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat());
+            if (remoteSize == null || remoteSize < 0) {
+                s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
+                        "url %s, obtained size = %s", url, remoteSize));
+                return new CheckUrlAnswer(false, remoteSize);
+            }
         }
         return new CheckUrlAnswer(checkResult, remoteSize);
     }

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java

@@ -347,7 +347,7 @@ private String postRemoteDownload(String jobId) {
             // and as such can be easily read.
 
             try (InputStream inputStream = td.getS3ObjectInputStream();) {
-                dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream));
+                dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream, false));
             }
             catch (IOException e) {
                 result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage();

utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java

@@ -25,10 +25,9 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 
 import org.apache.commons.compress.compressors.CompressorException;
-import org.apache.commons.compress.compressors.CompressorInputStream;
 import org.apache.commons.compress.compressors.CompressorStreamFactory;
 import org.apache.log4j.Logger;
 
@@ -57,7 +56,10 @@ public static int getVirtualSizeHeaderLocation() {
      * @param inputStream The QCOW2 object in stream format.
      * @return The virtual size of the QCOW2 object.
      */
-    public static long getVirtualSize(InputStream inputStream) throws IOException {
+    public static long getVirtualSize(InputStream inputStream, boolean isCompressed) throws IOException {
+        if (isCompressed) {
+            return getVirtualSizeFromInputStream(inputStream);
+        }
         byte[] bytes = new byte[VIRTUALSIZE_HEADER_LENGTH];
 
         if (inputStream.skip(VIRTUALSIZE_HEADER_LOCATION) != VIRTUALSIZE_HEADER_LOCATION) {
@@ -71,20 +73,13 @@ public static long getVirtualSize(InputStream inputStream) throws IOException {
         return NumbersUtil.bytesToLong(bytes);
     }
 
-    public static long getVirtualSize(String urlStr) {
-        InputStream inputStream = null;
-
+    private static long getVirtualSizeFromInputStream(InputStream inputStream) throws IOException {
         try {
-            URL url = new URL(urlStr);
-            BufferedInputStream bufferedInputStream = new BufferedInputStream(url.openStream());
-            inputStream = bufferedInputStream;
-
             try {
-                CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream);
-                inputStream = compressorInputStream;
+                BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream);
+                inputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream);
             } catch (CompressorException e) {
                 LOGGER.warn(e.getMessage());
-                inputStream = bufferedInputStream;
             }
 
             byte[] inputBytes = inputStream.readNBytes(VIRTUALSIZE_HEADER_LOCATION + VIRTUALSIZE_HEADER_LENGTH);
@@ -93,7 +88,7 @@ public static long getVirtualSize(String urlStr) {
             inputMagicBytes.put(inputBytes, 0, MAGIC_HEADER_LENGTH);
 
             ByteBuffer qcow2MagicBytes = ByteBuffer.allocate(MAGIC_HEADER_LENGTH);
-            qcow2MagicBytes.put("QFI".getBytes(Charset.forName("UTF-8")));
+            qcow2MagicBytes.put("QFI".getBytes(StandardCharsets.UTF_8));
             qcow2MagicBytes.put((byte)0xfb);
 
             long virtualSize = 0L;
@@ -105,12 +100,6 @@ public static long getVirtualSize(String urlStr) {
             }
 
             return virtualSize;
-        } catch (MalformedURLException e) {
-            LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
-            throw new IllegalArgumentException("Invalid URL: " + urlStr);
-        }  catch (IOException e) {
-            LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
-            throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
         } finally {
             if (inputStream != null) {
                 try {
@@ -121,4 +110,17 @@ public static long getVirtualSize(String urlStr) {
             }
         }
     }
+
+    public static long getVirtualSize(String urlStr) {
+        try {
+            URL url = new URL(urlStr);
+            return getVirtualSizeFromInputStream(url.openStream());
+        } catch (MalformedURLException e) {
+            LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
+            throw new IllegalArgumentException("Invalid URL: " + urlStr);
+        }  catch (IOException e) {
+            LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
+            throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
+        }
+    }
 }

utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java

@@ -24,7 +24,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -67,7 +67,7 @@ public void setup() {
         ByteBuffer byteBuffer = ByteBuffer.allocate(72);
 
         // Magic
-        byteBuffer.put("QFI".getBytes(Charset.forName("UTF-8")));
+        byteBuffer.put("QFI".getBytes(StandardCharsets.UTF_8));
         byteBuffer.put((byte)0xfb);
 
         // Version
@@ -116,6 +116,6 @@ public void getVirtualSizeHeaderLocation() {
 
     @Test
     public void getVirtualSizeTest() throws IOException {
-        assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream));
+        assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream, false));
     }
 }