feat(CLI): Add option for async download [TSI-2515] (#649)

phrase · Jul 3, 2024 · 976353a · 976353a
1 parent 4060b73
commit 976353a
Show file tree

Hide file tree

Showing 3 changed files with 131 additions and 32 deletions.
diff --git a/clients/cli/cmd/internal/pull.go b/clients/cli/cmd/internal/pull.go
@@ -2,10 +2,13 @@ package internal
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"time"
 
@@ -19,6 +22,8 @@ import (
 
 const (
 	timeoutInMinutes = 30 * time.Minute
+	asyncWaitTime    = 5 * time.Second
+	asyncRetryCount  = 360 // 30 minutes
 )
 
 var Config *phrase.Config
@@ -27,6 +32,7 @@ type PullCommand struct {
 	phrase.Config
 	Branch             string
 	UseLocalBranchName bool
+	Async              bool
 }
 
 var Auth context.Context
@@ -69,7 +75,7 @@ func (cmd *PullCommand) Run(config *phrase.Config) error {
 	}
 
 	for _, target := range targets {
-		err := target.Pull(client, cmd.Branch)
+		err := target.Pull(client, cmd.Branch, cmd.Async)
 		if err != nil {
 			return err
 		}
@@ -97,7 +103,7 @@ type PullParams struct {
 	LocaleID                  string `json:"locale_id"`
 }
 
-func (target *Target) Pull(client *phrase.APIClient, branch string) error {
+func (target *Target) Pull(client *phrase.APIClient, branch string, async bool) error {
 	if err := target.CheckPreconditions(); err != nil {
 		return err
 	}
@@ -118,7 +124,7 @@ func (target *Target) Pull(client *phrase.APIClient, branch string) error {
 			return err
 		}
 
-		err = target.DownloadAndWriteToFile(client, localeFile, branch)
+		err = target.DownloadAndWriteToFile(client, localeFile, branch, async)
 		if err != nil {
 			if openapiError, ok := err.(phrase.GenericOpenAPIError); ok {
 				print.Warn("API response: %s", openapiError.Body())
@@ -127,15 +133,13 @@ func (target *Target) Pull(client *phrase.APIClient, branch string) error {
 		} else {
 			print.Success("Downloaded %s to %s", localeFile.Message(), localeFile.RelPath())
 		}
-		if Debug {
-			fmt.Fprintln(os.Stderr, strings.Repeat("-", 10))
-		}
+		debugFprintln(strings.Repeat("-", 10))
 	}
 
 	return nil
 }
 
-func (target *Target) DownloadAndWriteToFile(client *phrase.APIClient, localeFile *LocaleFile, branch string) error {
+func (target *Target) DownloadAndWriteToFile(client *phrase.APIClient, localeFile *LocaleFile, branch string, async bool) error {
 	localVarOptionals := phrase.LocaleDownloadOpts{}
 
 	if target.Params != nil {
@@ -155,47 +159,134 @@ func (target *Target) DownloadAndWriteToFile(client *phrase.APIClient, localeFil
 		localVarOptionals.Tag = optional.EmptyString()
 	}
 
-	if Debug {
-		fmt.Fprintln(os.Stderr, "Target file pattern:", target.File)
-		fmt.Fprintln(os.Stderr, "Actual file path", localeFile.Path)
-		fmt.Fprintln(os.Stderr, "LocaleID", localeFile.ID)
-		fmt.Fprintln(os.Stderr, "ProjectID", target.ProjectID)
-		fmt.Fprintln(os.Stderr, "FileFormat", localVarOptionals.FileFormat)
-		fmt.Fprintln(os.Stderr, "ConvertEmoji", localVarOptionals.ConvertEmoji)
-		fmt.Fprintln(os.Stderr, "IncludeEmptyTranslations", localVarOptionals.IncludeEmptyTranslations)
-		fmt.Fprintln(os.Stderr, "KeepNotranslateTags", localVarOptionals.KeepNotranslateTags)
-		fmt.Fprintln(os.Stderr, "Tags", localVarOptionals.Tags)
-		fmt.Fprintln(os.Stderr, "Branch", localVarOptionals.Branch)
-		fmt.Fprintln(os.Stderr, "FormatOptions", localVarOptionals.FormatOptions)
+	debugFprintln("Target file pattern:", target.File)
+	debugFprintln("Actual file path", localeFile.Path)
+	debugFprintln("LocaleID", localeFile.ID)
+	debugFprintln("ProjectID", target.ProjectID)
+	debugFprintln("FileFormat", localVarOptionals.FileFormat)
+	debugFprintln("ConvertEmoji", localVarOptionals.ConvertEmoji)
+	debugFprintln("IncludeEmptyTranslations", localVarOptionals.IncludeEmptyTranslations)
+	debugFprintln("KeepNotranslateTags", localVarOptionals.KeepNotranslateTags)
+	debugFprintln("Tags", localVarOptionals.Tags)
+	debugFprintln("Branch", localVarOptionals.Branch)
+	debugFprintln("FormatOptions", localVarOptionals.FormatOptions)
+
+	if async {
+		return target.downloadAsynchronously(client, localeFile, localVarOptionals)
+	} else {
+		return target.downloadSynchronously(client, localeFile, localVarOptionals)
+	}
+}
+
+func (target *Target) downloadAsynchronously(client *phrase.APIClient, localeFile *LocaleFile, downloadOpts phrase.LocaleDownloadOpts) error {
+	localeDownloadCreateParams := asyncDownloadParams(downloadOpts)
+
+	localVarOptionals := phrase.LocaleDownloadCreateOpts{}
+	debugFprintln("Initiating async download...")
+	asyncDownload, _, err := client.LocaleDownloadsApi.LocaleDownloadCreate(Auth, target.ProjectID, localeFile.ID, localeDownloadCreateParams, &localVarOptionals)
+	if err != nil {
+		return err
+	}
+
+	for i := 0; asyncDownload.Status == "processing"; i++ {
+		debugFprintln("Waiting for the files to be exported...")
+		time.Sleep(asyncWaitTime)
+		debugFprintln("Checking if the download is ready...")
+		localVarOptionals := phrase.LocaleDownloadShowOpts{}
+		asyncDownload, _, err = client.LocaleDownloadsApi.LocaleDownloadShow(Auth, target.ProjectID, localeFile.ID, asyncDownload.Id, &localVarOptionals)
+		if err != nil {
+			return err
+		}
+		if i > asyncRetryCount {
+			return fmt.Errorf("download is taking too long")
+		}
+	}
+	if asyncDownload.Status == "completed" {
+		return downloadExportedLocale(asyncDownload.Result.Url, localeFile.Path)
 	}
+	return fmt.Errorf("download failed: %s", asyncDownload.Error)
+}
 
-	file, response, err := client.LocalesApi.LocaleDownload(Auth, target.ProjectID, localeFile.ID, &localVarOptionals)
+func (target *Target) downloadSynchronously(client *phrase.APIClient, localeFile *LocaleFile, downloadOpts phrase.LocaleDownloadOpts) error {
+	file, response, err := client.LocalesApi.LocaleDownload(Auth, target.ProjectID, localeFile.ID, &downloadOpts)
 	if err != nil {
 		if response.Rate.Remaining == 0 {
 			waitForRateLimit(response.Rate)
-			file, _, err = client.LocalesApi.LocaleDownload(Auth, target.ProjectID, localeFile.ID, &localVarOptionals)
+			file, _, err = client.LocalesApi.LocaleDownload(Auth, target.ProjectID, localeFile.ID, &downloadOpts)
 			if err != nil {
 				return err
 			}
 		} else {
 			return err
 		}
 	}
+	return copyToDestination(file, localeFile.Path)
+}
 
+func copyToDestination(file *os.File, path string) error {
 	var data []byte
-	if file != nil {
-		data, err = ioutil.ReadAll(file)
-		if err != nil {
-			return err
-		}
-		file.Close()
-		os.Remove(file.Name())
+	data, err := io.ReadAll(file)
+	if err != nil {
+		return err
 	}
+	file.Close()
+	os.Remove(file.Name())
 
-	err = ioutil.WriteFile(localeFile.Path, data, 0644)
+	err = os.WriteFile(path, data, 0644)
 	return err
 }
 
+func downloadExportedLocale(url string, localName string) error {
+	debugFprintln("Downloading file from ", url)
+	file, err := os.Create(localName)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	request, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return err
+	}
+	request.Header.Set("Authorization", "Bearer "+Config.Credentials.Token)
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	io.Copy(file, response.Body)
+	return nil
+}
+
+func asyncDownloadParams(localVarOptionals phrase.LocaleDownloadOpts) phrase.LocaleDownloadCreateParameters {
+	sourceFields := reflect.VisibleFields(reflect.TypeOf(localVarOptionals))
+	localeDownloadCreateParams := phrase.LocaleDownloadCreateParameters{}
+	targetFields := reflect.VisibleFields(reflect.TypeOf(localeDownloadCreateParams))
+
+	for i, targetField := range targetFields {
+		for _, sourceField := range sourceFields {
+			if targetField.Name == sourceField.Name {
+				sourceValue := reflect.ValueOf(localVarOptionals).FieldByName(sourceField.Name)
+				if sourceValue.MethodByName("IsSet").Call([]reflect.Value{})[0].Interface().(bool) {
+					targetValue := reflect.ValueOf(&localeDownloadCreateParams).Elem().Field(i)
+					sourceOptionalValue := sourceValue.MethodByName("Value").Call([]reflect.Value{})[0]
+					switch sourceField.Type {
+					case reflect.TypeOf((*optional.String)(nil)).Elem():
+						targetValue.Set(sourceOptionalValue)
+					case reflect.TypeOf((*optional.Bool)(nil)).Elem():
+						boolValue := sourceOptionalValue.Interface().(bool)
+						targetValue.Set(reflect.ValueOf(&boolValue))
+					case reflect.TypeOf((*optional.Interface)(nil)).Elem():
+						jsonValue, _ := json.Marshal(sourceOptionalValue.Interface())
+						json.Unmarshal(jsonValue, targetValue.Addr().Interface())
+					}
+				}
+				break
+			}
+		}
+	}
+	return localeDownloadCreateParams
+}
+
 func (target *Target) LocaleFiles() (LocaleFiles, error) {
 	files := []*LocaleFile{}
 
@@ -309,3 +400,9 @@ func createFile(path string) error {
 
 	return nil
 }
+
+func debugFprintln(a ...any) {
+	if Debug {
+		fmt.Fprintln(os.Stderr, a...)
+	}
+}
diff --git a/clients/cli/cmd/pull.go b/clients/cli/cmd/pull.go
@@ -20,6 +20,7 @@ func initPull() {
 			cmdPull := pull.PullCommand{
 				Branch:             params.GetString("branch"),
 				UseLocalBranchName: params.GetBool("use-local-branch-name"),
+				Async:              params.GetBool("async"),
 			}
 			err := cmdPull.Run(Config)
 			if err != nil {
@@ -31,5 +32,6 @@ func initPull() {
 
 	AddFlag(pullCmd, "string", "branch", "b", "branch", false)
 	AddFlag(pullCmd, "bool", "use-local-branch-name", "", "use local branch name", false)
+	AddFlag(pullCmd, "bool", "async", "a", "use asynchronous locale downloads (recommended for large number of keys)", false)
 	params.BindPFlags(pullCmd.Flags())
 }
diff --git a/clients/cli/go.sum b/clients/cli/go.sum
@@ -220,8 +220,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
-github.com/phrase/phrase-go/v3 v3.0.1 h1:A8H1Vmfg1BMEjKMhwJW0j6FRd/qTER+Je9JnXUfP7ks=
-github.com/phrase/phrase-go/v3 v3.0.1/go.mod h1:8vb6fBPJ45NbvJ9rqP8+Auai+NjmgiHdGPMPFreI8Xw=
+github.com/phrase/phrase-go/v3 v3.3.0 h1:kq2eFgKE6mUUZpud1KWsTa1RO4T+ztB2JI3DsCfqHog=
+github.com/phrase/phrase-go/v3 v3.3.0/go.mod h1:s0uOYiXLxKAYlaIS6TbKv3efkKFUlY4OB6OL+VgvK90=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=