-
-
Notifications
You must be signed in to change notification settings - Fork 2.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
net.http: implement http.download_file_with_progress/2, saving each c…
…hunk, as it is received, without growing the memory usage (#21633)
- Loading branch information
Showing
9 changed files
with
354 additions
and
85 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| module http | ||
|
|
||
| // Downloader is the interface that you have to implement, if you need to customise | ||
| // how download_file_with_progress works, and what output it produces while a file | ||
| // is downloaded. | ||
| pub interface Downloader { | ||
| mut: | ||
| // Called once, at the start of the streaming download. You can do setup here, | ||
| // like opening a target file, changing request.stop_copying_limit to a different value, | ||
| // if you need it. | ||
| on_start(mut request Request, path string) ! | ||
| // Called many times, once a chunk of data is received | ||
| on_chunk(request &Request, chunk []u8, already_received u64, expected u64) ! | ||
| // Called once, at the end of the streaming download. Do cleanup here, | ||
| // like closing a file (opened in on_start), reporting stats etc. | ||
| on_finish(request &Request, response &Response) ! | ||
| } | ||
|
|
||
| // DownloaderParams is similar to FetchConfig, but it also allows you to pass | ||
| // a `downloader: your_downloader_instance` parameter. | ||
| // See also http.SilentStreamingDownloader, and http.TerminalStreamingDownloader . | ||
| @[params] | ||
| pub struct DownloaderParams { | ||
| FetchConfig | ||
| pub mut: | ||
| downloader &Downloader = TerminalStreamingDownloader{} | ||
| } | ||
|
|
||
| // download_file_with_progress will save the URL `url` to the filepath `path` . | ||
| // Unlike download_file/2, it *does not* load the whole content in memory, but | ||
| // instead streams it chunk by chunk to the target `path`, as the chunks are received | ||
| // from the network. This makes it suitable for downloading big files, *without* increasing | ||
| // the memory consumption of your application. | ||
| // | ||
| // By default, it will also show a progress line, while the download happens. | ||
| // If you do not want a status line, you can call it like this: | ||
| // `http.download_file_with_progress(url, path, downloader: http.SilentStreamingDownloader{})`, | ||
| // or you can implement your own http.Downloader and pass that instead. | ||
| // | ||
| // Note: the returned response by this function, will have a truncated .body, after the first | ||
| // few KBs, because it does not accumulate all its data in memory, instead relying on the | ||
| // downloaders to save the received data chunk by chunk. You can parametrise this by | ||
| // using `stop_copying_limit:` but you need to pass a number that is big enough to fit | ||
| // at least all headers in the response, otherwise the parsing of the response at the end will | ||
| // fail, despite saving all the data in the file before that. The default is 65536 bytes. | ||
| pub fn download_file_with_progress(url string, path string, params DownloaderParams) !Response { | ||
| mut d := unsafe { params.downloader } | ||
| mut config := params.FetchConfig | ||
| config.url = url | ||
| config.user_ptr = voidptr(d) | ||
| config.on_progress_body = download_progres_cb | ||
| if config.stop_copying_limit == -1 { | ||
| // leave more than enough space for potential redirect headers | ||
| config.stop_copying_limit = 65536 | ||
| } | ||
| mut req := prepare(config)! | ||
| d.on_start(mut req, path)! | ||
| response := req.do()! | ||
| d.on_finish(req, response)! | ||
| return response | ||
| } | ||
|
|
||
| const zz = &Downloader(unsafe { nil }) | ||
|
|
||
| fn download_progres_cb(request &Request, chunk []u8, body_so_far u64, expected_size u64, status_code int) ! { | ||
| // TODO: remove this hack, when `unsafe { &Downloader( request.user_ptr ) }` works reliably, | ||
| // by just casting, without trying to promote the argument to the heap at all. | ||
| mut d := unsafe { http.zz } | ||
| pd := unsafe { &voidptr(&d) } | ||
| unsafe { | ||
| *pd = request.user_ptr | ||
| } | ||
| if status_code == 200 { | ||
| // ignore redirects, we are interested in the chunks of the final file: | ||
| d.on_chunk(request, chunk, body_so_far, expected_size)! | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| module http | ||
|
|
||
| import os | ||
|
|
||
| // SilentStreamingDownloader just saves the downloaded file chunks to the given path. | ||
| // It does *no reporting at all*. | ||
| // Note: the folder part of the path should already exist, and has to be writable. | ||
| pub struct SilentStreamingDownloader { | ||
| pub mut: | ||
| path string | ||
| f os.File | ||
| } | ||
|
|
||
| // on_start is called once at the start of the download. | ||
| pub fn (mut d SilentStreamingDownloader) on_start(mut request Request, path string) ! { | ||
| d.path = path | ||
| d.f = os.create(path)! | ||
| } | ||
|
|
||
| // on_chunk is called multiple times, once per chunk of received content. | ||
| pub fn (mut d SilentStreamingDownloader) on_chunk(request &Request, chunk []u8, already_received u64, expected u64) ! { | ||
| d.f.write(chunk)! | ||
| } | ||
|
|
||
| // on_finish is called once at the end of the download. | ||
| pub fn (mut d SilentStreamingDownloader) on_finish(request &Request, response &Response) ! { | ||
| d.f.close() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| module http | ||
|
|
||
| import time | ||
|
|
||
| // TerminalStreamingDownloader is the same as http.SilentStreamingDownloader, but produces a progress line on stdout. | ||
| pub struct TerminalStreamingDownloader { | ||
| SilentStreamingDownloader | ||
| mut: | ||
| start_time time.Time | ||
| past_time time.Time | ||
| past_received u64 | ||
| } | ||
|
|
||
| // on_start is called once at the start of the download. | ||
| pub fn (mut d TerminalStreamingDownloader) on_start(mut request Request, path string) ! { | ||
| d.SilentStreamingDownloader.on_start(mut request, path)! | ||
| d.start_time = time.now() | ||
| d.past_time = time.now() | ||
| } | ||
|
|
||
| // on_chunk is called multiple times, once per chunk of received content. | ||
| pub fn (mut d TerminalStreamingDownloader) on_chunk(request &Request, chunk []u8, already_received u64, expected u64) ! { | ||
| now := time.now() | ||
| elapsed := now - d.start_time | ||
| // delta_elapsed := now - d.past_time | ||
| // delta_bytes := already_received - d.past_received | ||
| d.past_time = now | ||
| d.past_received = already_received | ||
| ratio := f64(already_received) / f64(expected) | ||
| estimated := time.Duration(i64(f64(elapsed) / ratio)) | ||
| speed := f64(time.millisecond) * f64(already_received) / f64(elapsed) | ||
| elapsed_s := elapsed.seconds() | ||
| estimated_s := estimated.seconds() | ||
| eta_s := f64_max(estimated_s - elapsed_s, 0.0) | ||
|
|
||
| d.SilentStreamingDownloader.on_chunk(request, chunk, already_received, expected)! | ||
| print('\rDownloading to `${d.path}` ${100.0 * ratio:6.2f}%, ${f64(already_received) / (1024 * 1024):7.3f}/${f64(expected) / (1024 * 1024):-7.3f}MB, ${speed:6.0f}KB/s, elapsed: ${elapsed_s:6.0f}s, eta: ${eta_s:6.0f}s') | ||
| flush_stdout() | ||
| } | ||
|
|
||
| // on_finish is called once at the end of the download. | ||
| pub fn (mut d TerminalStreamingDownloader) on_finish(request &Request, response &Response) ! { | ||
| d.SilentStreamingDownloader.on_finish(request, response)! | ||
| println('') | ||
| flush_stdout() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.