From 21a9e973c3264c0e43a489590c435eab5dd0af85 Mon Sep 17 00:00:00 2001 From: JP Leibundguth Date: Tue, 23 Nov 2021 15:15:12 -0800 Subject: [PATCH] Experiment with 1MB buffer size to increase download speed Ref: https://www.amitph.com/java-read-write-large-files-efficiently/#FileChannel_and_Chunk_Transfer_using_BufferedInputStream --- .../api/consumer/fs/HollowFilesystemBlobRetriever.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/consumer/fs/HollowFilesystemBlobRetriever.java b/hollow/src/main/java/com/netflix/hollow/api/consumer/fs/HollowFilesystemBlobRetriever.java index 0583193dfb..ca600bd8c7 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/consumer/fs/HollowFilesystemBlobRetriever.java +++ b/hollow/src/main/java/com/netflix/hollow/api/consumer/fs/HollowFilesystemBlobRetriever.java @@ -36,6 +36,8 @@ import java.util.UUID; public class HollowFilesystemBlobRetriever implements HollowConsumer.BlobRetriever { + private static final int BUFFER_SIZE = 1048576; + private final Path blobStorePath; private final HollowConsumer.BlobRetriever fallbackBlobRetriever; private final boolean useExistingStaleSnapshot; @@ -83,7 +85,6 @@ public HollowFilesystemBlobRetriever(Path blobStorePath, HollowConsumer.BlobRetr this.fallbackBlobRetriever = fallbackBlobRetriever; this.useExistingStaleSnapshot = useExistingStaleSnapshot; this.optionalBlobParts = fallbackBlobRetriever == null ? null : fallbackBlobRetriever.configuredOptionalBlobParts(); - ensurePathExists(blobStorePath); } @@ -319,7 +320,7 @@ public InputStream getInputStream() throws IOException { InputStream is = remoteBlob.getInputStream(); OutputStream os = Files.newOutputStream(tempPath) ) { - byte buf[] = new byte[4096]; + byte buf[] = new byte[BUFFER_SIZE]; int n; while (-1 != (n = is.read(buf))) os.write(buf, 0, n); @@ -337,7 +338,7 @@ public File getFile() throws IOException { InputStream is = remoteBlob.getInputStream(); OutputStream os = Files.newOutputStream(tempPath) ) { - byte buf[] = new byte[4096]; + byte buf[] = new byte[BUFFER_SIZE]; int n; while (-1 != (n = is.read(buf))) os.write(buf, 0, n); @@ -364,7 +365,7 @@ public OptionalBlobPartInput getOptionalBlobPartInputs() throws IOException { InputStream is = entry.getValue(); OutputStream os = Files.newOutputStream(tempPath) ) { - byte buf[] = new byte[4096]; + byte buf[] = new byte[BUFFER_SIZE]; int n; while (-1 != (n = is.read(buf, 0, buf.length))) os.write(buf, 0, n);