Skip to content

Commit

Permalink
refactor: extract processing single link to function
Browse files Browse the repository at this point in the history
  • Loading branch information
michalczmiel committed Jan 25, 2024
1 parent 4d56c5b commit ed3ad81
Showing 1 changed file with 49 additions and 41 deletions.
90 changes: 49 additions & 41 deletions internal/images.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,47 +13,43 @@ type Parameters struct {
Referer string
}

func downloadWorker(wg *sync.WaitGroup, httClient HttpClent, parameters *Parameters, linksToProcess <-chan string, results chan<- DownloadResult) {
for link := range linksToProcess {
fileName, err := GetFileNameFromUrl(link)
if err != nil {
results <- DownloadResult{Url: link, Err: err}
}

filePath := path.Join(parameters.Directory, fileName)

var referer string
if parameters.Referer == "" {
referer = getRootUrl(link)
} else {
referer = parameters.Referer
}

response, err := httClient.Request(link, map[string]string{
"User-Agent": parameters.UserAgent,
"Referer": referer,
})
if err != nil {
results <- DownloadResult{Url: link, Err: err}
}

contentType := response.Header.Get("Content-Type")
err = validateContentType(contentType, parameters.ImageTypes)
if err != nil {
results <- DownloadResult{Url: link, Err: err}
}

filePath = addExtensionIfMissing(filePath, contentType)

err = SaveToFile(response.Body, filePath)
if err != nil {
results <- DownloadResult{Url: link, Err: err}
}

results <- DownloadResult{Url: link, Err: nil}

wg.Done()
func downloadImage(link string, httClient HttpClent, parameters *Parameters) error {
fileName, err := GetFileNameFromUrl(link)
if err != nil {
return err
}

filePath := path.Join(parameters.Directory, fileName)

var referer string
if parameters.Referer == "" {
referer = getRootUrl(link)
} else {
referer = parameters.Referer
}

response, err := httClient.Request(link, map[string]string{
"User-Agent": parameters.UserAgent,
"Referer": referer,
})
if err != nil {
return err
}

contentType := response.Header.Get("Content-Type")
err = validateContentType(contentType, parameters.ImageTypes)
if err != nil {
return err
}

filePath = addExtensionIfMissing(filePath, contentType)

err = SaveToFile(response.Body, filePath)
if err != nil {
return err
}

return nil
}

type DownloadResult struct {
Expand All @@ -76,7 +72,19 @@ func DownloadImages(links []string, httClient HttpClent, parameters *Parameters)
wg.Add(len(links))

for i := 0; i < parameters.Concurrent; i++ {
go downloadWorker(&wg, httClient, parameters, linksToProcess, results)
go func() {
for link := range linksToProcess {
err := downloadImage(link, httClient, parameters)

if err != nil {
results <- DownloadResult{Url: link, Err: err}
} else {
results <- DownloadResult{Url: link, Err: nil}
}

wg.Done()
}
}()
}

// wait for all workers to finish
Expand Down

0 comments on commit ed3ad81

Please sign in to comment.