Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enforce upload size limit on putResourceFromUrl api #8562

Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
d4bcff3
Check content-length of resource at url
tylerjmchugh Dec 13, 2024
a0e2fdc
Remove unnecessary throw
tylerjmchugh Dec 13, 2024
b5b6dbf
Fallback to InputStream.available() if content length is not found or…
tylerjmchugh Dec 17, 2024
2b54b4d
Fallback maxUploadSize for unit tests
tylerjmchugh Dec 17, 2024
4d6a9e7
Fix default value always used
tylerjmchugh Dec 18, 2024
263d091
Remove content-length check as it cannot be trusted
tylerjmchugh Dec 18, 2024
00770fa
Download file to a temp file to check size
tylerjmchugh Dec 20, 2024
f4232fa
Add back the content-length check as a preliminary check
tylerjmchugh Dec 30, 2024
c5193fa
Stream file instead of using temp file
tylerjmchugh Dec 30, 2024
cd7418e
Rollback for JCloud
tylerjmchugh Dec 30, 2024
e73fb98
Fix rollback for JCloud
tylerjmchugh Dec 30, 2024
c0b921b
Fix abstract store and implement jcloud rollback
tylerjmchugh Jan 2, 2025
5878548
Fix typo and remove unused import
tylerjmchugh Jan 2, 2025
6e5d0d0
Fix unit tests
tylerjmchugh Jan 2, 2025
fba9139
Fix tests (mock response code)
tylerjmchugh Jan 2, 2025
afaef1b
Fix unit test, refactor, and add docs
tylerjmchugh Jan 2, 2025
bbc9526
Fix exception handling and use bounded input stream instead of custom…
tylerjmchugh Jan 3, 2025
a6b1d0e
Improvements
tylerjmchugh Jan 6, 2025
3800bd7
Add documentation
tylerjmchugh Jan 6, 2025
9a2798f
Rename exception
tylerjmchugh Jan 6, 2025
da12ecb
Remove unneeded changes
tylerjmchugh Jan 6, 2025
0c27ae1
Update docs
tylerjmchugh Jan 6, 2025
b3d21f6
Fix comment
tylerjmchugh Jan 7, 2025
d557358
Update exception handling
tylerjmchugh Jan 9, 2025
a539228
Update jcloud exception handling and comments
tylerjmchugh Jan 10, 2025
4315340
Add file header
tylerjmchugh Jan 13, 2025
084fbf7
Add comment
tylerjmchugh Jan 13, 2025
c9778a4
Fix whitespace
tylerjmchugh Jan 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import jeeves.server.context.ServiceContext;
import org.apache.commons.io.FilenameUtils;
import org.fao.geonet.ApplicationContextHolder;
import org.fao.geonet.api.exception.GeonetMaxUploadSizeExceededException;
import org.fao.geonet.api.exception.NotAllowedException;
import org.fao.geonet.api.exception.ResourceNotFoundException;
import org.fao.geonet.domain.AbstractMetadata;
Expand All @@ -35,13 +36,14 @@
import org.fao.geonet.kernel.AccessManager;
import org.fao.geonet.kernel.datamanager.IMetadataUtils;
import org.fao.geonet.repository.MetadataRepository;
import org.fao.geonet.util.FileUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.web.multipart.MultipartFile;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
Expand All @@ -59,6 +61,9 @@ public abstract class AbstractStore implements Store {
protected static final String RESOURCE_MANAGEMENT_EXTERNAL_PROPERTIES_ESCAPED_SEPARATOR = "\\:";
private static final Logger log = LoggerFactory.getLogger(AbstractStore.class);

@Value("${api.params.maxUploadSize}")
private int maxUploadSize;

@Override
public final List<MetadataResource> getResources(final ServiceContext context, final String metadataUuid, final Sort sort,
final String filter) throws Exception {
Expand Down Expand Up @@ -157,7 +162,7 @@ protected int canDownload(ServiceContext context, String metadataUuid, MetadataR
return metadataId;
}

protected String getFilenameFromHeader(final URL fileUrl) throws IOException {
protected String getFilenameFromHeader(final URL fileUrl) {
HttpURLConnection connection = null;
try {
connection = (HttpURLConnection) fileUrl.openConnection();
Expand Down Expand Up @@ -230,7 +235,18 @@ public final MetadataResource putResource(ServiceContext context, String metadat
if (filename == null) {
filename = getFilenameFromUrl(fileUrl);
}
return putResource(context, metadataUuid, filename, fileUrl.openStream(), null, visibility, approved);
try (InputStream is = fileUrl.openStream()) {
int availableBytes = is.available();
if (availableBytes > maxUploadSize) {
throw new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException")
.withMessageKey("exception.maxUploadSizeExceeded",
new String[]{FileUtil.humanizeFileSize(maxUploadSize)})
.withDescriptionKey("exception.maxUploadSizeExceeded.description",
new String[]{FileUtil.humanizeFileSize(availableBytes),
FileUtil.humanizeFileSize(maxUploadSize)});
}
return putResource(context, metadataUuid, filename, is, null, visibility, approved);
}
}

@Override
Expand Down
2 changes: 2 additions & 0 deletions core/src/test/resources/WEB-INF/config.properties
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,7 @@ es.index.checker.interval=0/5 * * * * ?

thesaurus.cache.maxsize=400000

api.params.maxUploadSize=100000000

language.default=eng
language.forceDefault=false
Loading