Enforce upload size limit on putResourceFromUrl api

core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java

@@ -26,7 +26,9 @@
 
 import jeeves.server.context.ServiceContext;
 import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.input.BoundedInputStream;
 import org.fao.geonet.ApplicationContextHolder;
+import org.fao.geonet.api.exception.GeonetMaxUploadSizeExceededException;
 import org.fao.geonet.api.exception.NotAllowedException;
 import org.fao.geonet.api.exception.ResourceNotFoundException;
 import org.fao.geonet.domain.AbstractMetadata;
@@ -35,13 +37,15 @@
 import org.fao.geonet.kernel.AccessManager;
 import org.fao.geonet.kernel.datamanager.IMetadataUtils;
 import org.fao.geonet.repository.MetadataRepository;
+import org.fao.geonet.util.FileUtil;
+import org.fao.geonet.util.LimitedInputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.ApplicationContext;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.BufferedInputStream;
-import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -59,6 +63,9 @@ public abstract class AbstractStore implements Store {
     protected static final String RESOURCE_MANAGEMENT_EXTERNAL_PROPERTIES_ESCAPED_SEPARATOR = "\\:";
     private static final Logger log = LoggerFactory.getLogger(AbstractStore.class);
 
+    @Value("${api.params.maxUploadSize}")
+    protected int maxUploadSize;
+
     @Override
     public final List<MetadataResource> getResources(final ServiceContext context, final String metadataUuid, final Sort sort,
             final String filter) throws Exception {
@@ -157,7 +164,7 @@ protected int canDownload(ServiceContext context, String metadataUuid, MetadataR
         return metadataId;
     }
 
-    protected String getFilenameFromHeader(final URL fileUrl) throws IOException {
+    protected String getFilenameFromHeader(final URL fileUrl) {
         HttpURLConnection connection = null;
         try {
             connection = (HttpURLConnection) fileUrl.openConnection();
@@ -180,6 +187,23 @@ protected String getFilenameFromHeader(final URL fileUrl) throws IOException {
         }
     }
 
+    protected long getContentLengthFromHeader(final URL fileUrl) {
+        HttpURLConnection connection = null;
+        try {
+            connection = (HttpURLConnection) fileUrl.openConnection();
+            connection.setRequestMethod("HEAD");
+            connection.connect();
+            return connection.getContentLengthLong();
+        } catch (Exception e) {
+            log.error("Error retrieving resource content length from header", e);
+            return -1;
+        } finally {
+            if (connection != null) {
+                connection.disconnect();
+            }
+        }
+    }
+
     protected String getFilenameFromUrl(final URL fileUrl) {
         String fileName = FilenameUtils.getName(fileUrl.getPath());
         if (fileName.contains("?")) {
@@ -230,7 +254,20 @@ public final MetadataResource putResource(ServiceContext context, String metadat
         if (filename == null) {
             filename = getFilenameFromUrl(fileUrl);
         }
-        return putResource(context, metadataUuid, filename, fileUrl.openStream(), null, visibility, approved);
+
+        long contentLength = getContentLengthFromHeader(fileUrl);
+        if (contentLength > maxUploadSize) {
+            throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException")
+                .withMessageKey("exception.maxUploadSizeExceeded",
+                    new String[]{FileUtil.humanizeFileSize(maxUploadSize)})
+                .withDescriptionKey("exception.maxUploadSizeExceeded.description",
+                    new String[]{FileUtil.humanizeFileSize(contentLength),
+                        FileUtil.humanizeFileSize(maxUploadSize)});
+        }
+
+        try (InputStream is = new LimitedInputStream(fileUrl.openStream(), maxUploadSize+1)) {
+            return putResource(context, metadataUuid, filename, is, null, visibility, approved);
+        }
     }
 
     @Override

core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java

@@ -26,6 +26,7 @@
 package org.fao.geonet.api.records.attachments;
 
 import jeeves.server.context.ServiceContext;
+import org.fao.geonet.api.exception.GeonetMaxUploadSizeExceededException;
 import org.fao.geonet.api.exception.ResourceAlreadyExistException;
 import org.fao.geonet.api.exception.ResourceNotFoundException;
 import org.fao.geonet.constants.Geonet;
@@ -35,6 +36,8 @@
 import org.fao.geonet.kernel.GeonetworkDataDirectory;
 import org.fao.geonet.kernel.setting.SettingManager;
 import org.fao.geonet.lib.Lib;
+import org.fao.geonet.util.FileUtil;
+import org.fao.geonet.util.LimitedInputStream;
 import org.fao.geonet.utils.IO;
 import org.fao.geonet.utils.Log;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -203,6 +206,14 @@ public MetadataResource putResource(final ServiceContext context, final String m
         checkResourceId(filename);
         Path filePath = getPath(context, metadataId, visibility, filename, approved);
         Files.copy(is, filePath, StandardCopyOption.REPLACE_EXISTING);
+        if (is instanceof LimitedInputStream && ((LimitedInputStream) is).isLimitReached()) {
+            Files.deleteIfExists(filePath);
+            throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException")
+                .withMessageKey("exception.maxUploadSizeExceeded",
+                    new String[]{FileUtil.humanizeFileSize(maxUploadSize)})
+                .withDescriptionKey("exception.maxUploadSizeExceededUnknownSize.description",
+                    new String[]{FileUtil.humanizeFileSize(maxUploadSize)});
+        }
         if (changeDate != null) {
             IO.touch(filePath, FileTime.from(changeDate.getTime(), TimeUnit.MILLISECONDS));
         }

core/src/main/java/org/fao/geonet/util/LimitedInputStream.java

@@ -0,0 +1,102 @@
+package org.fao.geonet.util;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class LimitedInputStream extends InputStream {
+    private final InputStream in;
+    private final long max;
+    private long pos;
+    private long mark;
+    private boolean propagateClose;
+
+    public LimitedInputStream(InputStream in, long size) {
+        this.pos = 0L;
+        this.mark = -1L;
+        this.propagateClose = true;
+        this.max = size;
+        this.in = in;
+    }
+
+    public LimitedInputStream(InputStream in) {
+        this(in, -1L);
+    }
+
+    public int read() throws IOException {
+        if (this.max >= 0L && this.pos >= this.max) {
+            return -1;
+        } else {
+            int result = this.in.read();
+            ++this.pos;
+            return result;
+        }
+    }
+
+    public int read(byte[] b) throws IOException {
+        return this.read(b, 0, b.length);
+    }
+
+    public int read(byte[] b, int off, int len) throws IOException {
+        if (this.max >= 0L && this.pos >= this.max) {
+            return -1;
+        } else {
+            long maxRead = this.max >= 0L ? Math.min((long)len, this.max - this.pos) : (long)len;
+            int bytesRead = this.in.read(b, off, (int)maxRead);
+            if (bytesRead == -1) {
+                return -1;
+            } else {
+                this.pos += (long)bytesRead;
+                return bytesRead;
+            }
+        }
+    }
+
+    public long skip(long n) throws IOException {
+        long toSkip = this.max >= 0L ? Math.min(n, this.max - this.pos) : n;
+        long skippedBytes = this.in.skip(toSkip);
+        this.pos += skippedBytes;
+        return skippedBytes;
+    }
+
+    public int available() throws IOException {
+        return this.max >= 0L && this.pos >= this.max ? 0 : this.in.available();
+    }
+
+    public boolean isLimitReached() {
+        return this.pos >= this.max;
+    }
+
+    public String toString() {
+        return this.in.toString();
+    }
+
+    public void close() throws IOException {
+        if (this.propagateClose) {
+            this.in.close();
+        }
+
+    }
+
+    public synchronized void reset() throws IOException {
+        this.in.reset();
+        this.pos = this.mark;
+    }
+
+    public synchronized void mark(int readlimit) {
+        this.in.mark(readlimit);
+        this.mark = this.pos;
+    }
+
+    public boolean markSupported() {
+        return this.in.markSupported();
+    }
+
+    public boolean isPropagateClose() {
+        return this.propagateClose;
+    }
+
+    public void setPropagateClose(boolean propagateClose) {
+        this.propagateClose = propagateClose;
+    }
+}

core/src/test/resources/WEB-INF/config.properties

@@ -20,5 +20,7 @@ es.index.checker.interval=0/5 * * * * ?
 
 thesaurus.cache.maxsize=400000
 
+api.params.maxUploadSize=100000000
+
 language.default=eng
 language.forceDefault=false

core/src/test/resources/org/fao/geonet/api/Messages.properties

@@ -178,6 +178,7 @@ api.exception.unsatisfiedRequestParameter=Unsatisfied request parameter
 api.exception.unsatisfiedRequestParameter.description=Unsatisfied request parameter.
 exception.maxUploadSizeExceeded=Maximum upload size of {0} exceeded.
 exception.maxUploadSizeExceeded.description=The request was rejected because its size ({0}) exceeds the configured maximum ({1}).
+exception.maxUploadSizeExceededUnknownSize.description=The request was rejected because its size exceeds the configured maximum ({0}).
 exception.resourceNotFound.metadata=Metadata not found
 exception.resourceNotFound.metadata.description=Metadata with UUID ''{0}'' not found.
 exception.resourceNotFound.resource=Metadata resource ''{0}'' not found

core/src/test/resources/org/fao/geonet/api/Messages_fre.properties

@@ -172,6 +172,7 @@ api.exception.unsatisfiedRequestParameter=Param\u00E8tre de demande non satisfai
 api.exception.unsatisfiedRequestParameter.description=Param\u00E8tre de demande non satisfait.
 exception.maxUploadSizeExceeded=La taille maximale du t\u00E9l\u00E9chargement de {0} a \u00E9t\u00E9 exc\u00E9d\u00E9e.
 exception.maxUploadSizeExceeded.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille ({0}) exc\u00E8de le maximum configur\u00E9 ({1}).
+exception.maxUploadSizeExceededUnknownSize.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille exc\u00E8de le maximum configur\u00E9 ({0}).
 exception.resourceNotFound.metadata=Fiches introuvables
 exception.resourceNotFound.metadata.description=La fiche ''{0}'' est introuvable.
 exception.resourceNotFound.resource=Ressource ''{0}'' introuvable

datastorages/jcloud/src/main/java/org/fao/geonet/api/records/attachments/JCloudStore.java

@@ -30,6 +30,7 @@
 
 import org.apache.commons.collections.MapUtils;
 import org.fao.geonet.ApplicationContextHolder;
+import org.fao.geonet.api.exception.GeonetMaxUploadSizeExceededException;
 import org.fao.geonet.api.exception.ResourceNotFoundException;
 import org.fao.geonet.constants.Geonet;
 import org.fao.geonet.domain.MetadataResource;
@@ -41,6 +42,8 @@
 import org.fao.geonet.languages.IsoLanguagesMapper;
 import org.fao.geonet.lib.Lib;
 import org.fao.geonet.resources.JCloudConfiguration;
+import org.fao.geonet.util.FileUtil;
+import org.fao.geonet.util.LimitedInputStream;
 import org.fao.geonet.utils.IO;
 import org.fao.geonet.utils.Log;
 import org.jclouds.blobstore.ContainerNotFoundException;
@@ -312,6 +315,14 @@ protected MetadataResource putResource(final ServiceContext context, final Strin
                 jCloudConfiguration.getClient().getBlobStore().putBlob(jCloudConfiguration.getContainerName(), blob, multipart());
                 Blob blobResults = jCloudConfiguration.getClient().getBlobStore().getBlob(jCloudConfiguration.getContainerName(), key);
 
+                if (is instanceof LimitedInputStream && ((LimitedInputStream) is).isLimitExceeded()) {
+                    delResource(context, metadataUuid, visibility, filename, approved);
+                    throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException")
+                        .withMessageKey("exception.maxUploadSizeExceeded",
+                            new String[]{FileUtil.humanizeFileSize(maxUploadSize)})
+                        .withDescriptionKey("exception.maxUploadSizeExceededUnknownSize.description",
+                            new String[]{FileUtil.humanizeFileSize(maxUploadSize)});
+                }
                 return createResourceDescription(context, metadataUuid, visibility, filename, blobResults.getMetadata(), metadataId, approved);
             } finally {
                 locks.remove(key);

web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages.properties

@@ -183,6 +183,7 @@ api.exception.unsatisfiedRequestParameter=Unsatisfied request parameter
 api.exception.unsatisfiedRequestParameter.description=Unsatisfied request parameter.
 exception.maxUploadSizeExceeded=Maximum upload size of {0} exceeded.
 exception.maxUploadSizeExceeded.description=The request was rejected because its size ({0}) exceeds the configured maximum ({1}).
+exception.maxUploadSizeExceededUnknownSize.description=The request was rejected because its size exceeds the configured maximum ({0}).
 exception.resourceNotFound.metadata=Metadata not found
 exception.resourceNotFound.metadata.description=Metadata with UUID ''{0}'' not found.
 exception.resourceNotFound.resource=Metadata resource ''{0}'' not found

web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages_fre.properties

@@ -177,6 +177,7 @@ api.exception.unsatisfiedRequestParameter=Param\u00E8tre de demande non satisfai
 api.exception.unsatisfiedRequestParameter.description=Param\u00E8tre de demande non satisfait.
 exception.maxUploadSizeExceeded=La taille maximale du t\u00E9l\u00E9chargement de {0} a \u00E9t\u00E9 exc\u00E9d\u00E9e.
 exception.maxUploadSizeExceeded.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille ({0}) exc\u00E8de le maximum configur\u00E9 ({1}).
+exception.maxUploadSizeExceededUnknownSize.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille exc\u00E8de le maximum configur\u00E9 ({0}).
 exception.resourceNotFound.metadata=Fiches introuvables
 exception.resourceNotFound.metadata.description=La fiche ''{0}'' est introuvable.
 exception.resourceNotFound.resource=Ressource ''{0}'' introuvable