diff --git a/clients/cli/cmd/internal/pull.go b/clients/cli/cmd/internal/pull.go index 4a43d6cf..de076d1f 100644 --- a/clients/cli/cmd/internal/pull.go +++ b/clients/cli/cmd/internal/pull.go @@ -2,10 +2,13 @@ package internal import ( "context" + "encoding/json" "fmt" - "io/ioutil" + "io" + "net/http" "os" "path/filepath" + "reflect" "strings" "time" @@ -19,6 +22,8 @@ import ( const ( timeoutInMinutes = 30 * time.Minute + asyncWaitTime = 5 * time.Second + asyncRetryCount = 360 // 30 minutes ) var Config *phrase.Config @@ -27,6 +32,7 @@ type PullCommand struct { phrase.Config Branch string UseLocalBranchName bool + Async bool } var Auth context.Context @@ -69,7 +75,7 @@ func (cmd *PullCommand) Run(config *phrase.Config) error { } for _, target := range targets { - err := target.Pull(client, cmd.Branch) + err := target.Pull(client, cmd.Branch, cmd.Async) if err != nil { return err } @@ -97,7 +103,7 @@ type PullParams struct { LocaleID string `json:"locale_id"` } -func (target *Target) Pull(client *phrase.APIClient, branch string) error { +func (target *Target) Pull(client *phrase.APIClient, branch string, async bool) error { if err := target.CheckPreconditions(); err != nil { return err } @@ -118,7 +124,7 @@ func (target *Target) Pull(client *phrase.APIClient, branch string) error { return err } - err = target.DownloadAndWriteToFile(client, localeFile, branch) + err = target.DownloadAndWriteToFile(client, localeFile, branch, async) if err != nil { if openapiError, ok := err.(phrase.GenericOpenAPIError); ok { print.Warn("API response: %s", openapiError.Body()) @@ -127,15 +133,13 @@ func (target *Target) Pull(client *phrase.APIClient, branch string) error { } else { print.Success("Downloaded %s to %s", localeFile.Message(), localeFile.RelPath()) } - if Debug { - fmt.Fprintln(os.Stderr, strings.Repeat("-", 10)) - } + debugFprintln(strings.Repeat("-", 10)) } return nil } -func (target *Target) DownloadAndWriteToFile(client *phrase.APIClient, localeFile *LocaleFile, branch string) error { +func (target *Target) DownloadAndWriteToFile(client *phrase.APIClient, localeFile *LocaleFile, branch string, async bool) error { localVarOptionals := phrase.LocaleDownloadOpts{} if target.Params != nil { @@ -155,25 +159,60 @@ func (target *Target) DownloadAndWriteToFile(client *phrase.APIClient, localeFil localVarOptionals.Tag = optional.EmptyString() } - if Debug { - fmt.Fprintln(os.Stderr, "Target file pattern:", target.File) - fmt.Fprintln(os.Stderr, "Actual file path", localeFile.Path) - fmt.Fprintln(os.Stderr, "LocaleID", localeFile.ID) - fmt.Fprintln(os.Stderr, "ProjectID", target.ProjectID) - fmt.Fprintln(os.Stderr, "FileFormat", localVarOptionals.FileFormat) - fmt.Fprintln(os.Stderr, "ConvertEmoji", localVarOptionals.ConvertEmoji) - fmt.Fprintln(os.Stderr, "IncludeEmptyTranslations", localVarOptionals.IncludeEmptyTranslations) - fmt.Fprintln(os.Stderr, "KeepNotranslateTags", localVarOptionals.KeepNotranslateTags) - fmt.Fprintln(os.Stderr, "Tags", localVarOptionals.Tags) - fmt.Fprintln(os.Stderr, "Branch", localVarOptionals.Branch) - fmt.Fprintln(os.Stderr, "FormatOptions", localVarOptionals.FormatOptions) + debugFprintln("Target file pattern:", target.File) + debugFprintln("Actual file path", localeFile.Path) + debugFprintln("LocaleID", localeFile.ID) + debugFprintln("ProjectID", target.ProjectID) + debugFprintln("FileFormat", localVarOptionals.FileFormat) + debugFprintln("ConvertEmoji", localVarOptionals.ConvertEmoji) + debugFprintln("IncludeEmptyTranslations", localVarOptionals.IncludeEmptyTranslations) + debugFprintln("KeepNotranslateTags", localVarOptionals.KeepNotranslateTags) + debugFprintln("Tags", localVarOptionals.Tags) + debugFprintln("Branch", localVarOptionals.Branch) + debugFprintln("FormatOptions", localVarOptionals.FormatOptions) + + if async { + return target.downloadAsynchronously(client, localeFile, localVarOptionals) + } else { + return target.downloadSynchronously(client, localeFile, localVarOptionals) + } +} + +func (target *Target) downloadAsynchronously(client *phrase.APIClient, localeFile *LocaleFile, downloadOpts phrase.LocaleDownloadOpts) error { + localeDownloadCreateParams := asyncDownloadParams(downloadOpts) + + localVarOptionals := phrase.LocaleDownloadCreateOpts{} + debugFprintln("Initiating async download...") + asyncDownload, _, err := client.LocaleDownloadsApi.LocaleDownloadCreate(Auth, target.ProjectID, localeFile.ID, localeDownloadCreateParams, &localVarOptionals) + if err != nil { + return err + } + + for i := 0; asyncDownload.Status == "processing"; i++ { + debugFprintln("Waiting for the files to be exported...") + time.Sleep(asyncWaitTime) + debugFprintln("Checking if the download is ready...") + localVarOptionals := phrase.LocaleDownloadShowOpts{} + asyncDownload, _, err = client.LocaleDownloadsApi.LocaleDownloadShow(Auth, target.ProjectID, localeFile.ID, asyncDownload.Id, &localVarOptionals) + if err != nil { + return err + } + if i > asyncRetryCount { + return fmt.Errorf("download is taking too long") + } + } + if asyncDownload.Status == "completed" { + return downloadExportedLocale(asyncDownload.Result.Url, localeFile.Path) } + return fmt.Errorf("download failed: %s", asyncDownload.Error) +} - file, response, err := client.LocalesApi.LocaleDownload(Auth, target.ProjectID, localeFile.ID, &localVarOptionals) +func (target *Target) downloadSynchronously(client *phrase.APIClient, localeFile *LocaleFile, downloadOpts phrase.LocaleDownloadOpts) error { + file, response, err := client.LocalesApi.LocaleDownload(Auth, target.ProjectID, localeFile.ID, &downloadOpts) if err != nil { if response.Rate.Remaining == 0 { waitForRateLimit(response.Rate) - file, _, err = client.LocalesApi.LocaleDownload(Auth, target.ProjectID, localeFile.ID, &localVarOptionals) + file, _, err = client.LocalesApi.LocaleDownload(Auth, target.ProjectID, localeFile.ID, &downloadOpts) if err != nil { return err } @@ -181,21 +220,73 @@ func (target *Target) DownloadAndWriteToFile(client *phrase.APIClient, localeFil return err } } + return copyToDestination(file, localeFile.Path) +} +func copyToDestination(file *os.File, path string) error { var data []byte - if file != nil { - data, err = ioutil.ReadAll(file) - if err != nil { - return err - } - file.Close() - os.Remove(file.Name()) + data, err := io.ReadAll(file) + if err != nil { + return err } + file.Close() + os.Remove(file.Name()) - err = ioutil.WriteFile(localeFile.Path, data, 0644) + err = os.WriteFile(path, data, 0644) return err } +func downloadExportedLocale(url string, localName string) error { + debugFprintln("Downloading file from ", url) + file, err := os.Create(localName) + if err != nil { + return err + } + defer file.Close() + request, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + request.Header.Set("Authorization", "Bearer "+Config.Credentials.Token) + response, err := http.DefaultClient.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + io.Copy(file, response.Body) + return nil +} + +func asyncDownloadParams(localVarOptionals phrase.LocaleDownloadOpts) phrase.LocaleDownloadCreateParameters { + sourceFields := reflect.VisibleFields(reflect.TypeOf(localVarOptionals)) + localeDownloadCreateParams := phrase.LocaleDownloadCreateParameters{} + targetFields := reflect.VisibleFields(reflect.TypeOf(localeDownloadCreateParams)) + + for i, targetField := range targetFields { + for _, sourceField := range sourceFields { + if targetField.Name == sourceField.Name { + sourceValue := reflect.ValueOf(localVarOptionals).FieldByName(sourceField.Name) + if sourceValue.MethodByName("IsSet").Call([]reflect.Value{})[0].Interface().(bool) { + targetValue := reflect.ValueOf(&localeDownloadCreateParams).Elem().Field(i) + sourceOptionalValue := sourceValue.MethodByName("Value").Call([]reflect.Value{})[0] + switch sourceField.Type { + case reflect.TypeOf((*optional.String)(nil)).Elem(): + targetValue.Set(sourceOptionalValue) + case reflect.TypeOf((*optional.Bool)(nil)).Elem(): + boolValue := sourceOptionalValue.Interface().(bool) + targetValue.Set(reflect.ValueOf(&boolValue)) + case reflect.TypeOf((*optional.Interface)(nil)).Elem(): + jsonValue, _ := json.Marshal(sourceOptionalValue.Interface()) + json.Unmarshal(jsonValue, targetValue.Addr().Interface()) + } + } + break + } + } + } + return localeDownloadCreateParams +} + func (target *Target) LocaleFiles() (LocaleFiles, error) { files := []*LocaleFile{} @@ -309,3 +400,9 @@ func createFile(path string) error { return nil } + +func debugFprintln(a ...any) { + if Debug { + fmt.Fprintln(os.Stderr, a...) + } +} diff --git a/clients/cli/cmd/pull.go b/clients/cli/cmd/pull.go index 81d7f7d6..83d75923 100644 --- a/clients/cli/cmd/pull.go +++ b/clients/cli/cmd/pull.go @@ -20,6 +20,7 @@ func initPull() { cmdPull := pull.PullCommand{ Branch: params.GetString("branch"), UseLocalBranchName: params.GetBool("use-local-branch-name"), + Async: params.GetBool("async"), } err := cmdPull.Run(Config) if err != nil { @@ -31,5 +32,6 @@ func initPull() { AddFlag(pullCmd, "string", "branch", "b", "branch", false) AddFlag(pullCmd, "bool", "use-local-branch-name", "", "use local branch name", false) + AddFlag(pullCmd, "bool", "async", "a", "use asynchronous locale downloads (recommended for large number of keys)", false) params.BindPFlags(pullCmd.Flags()) } diff --git a/clients/cli/go.sum b/clients/cli/go.sum index 0025b276..8a0936e6 100644 --- a/clients/cli/go.sum +++ b/clients/cli/go.sum @@ -220,8 +220,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/phrase/phrase-go/v3 v3.0.1 h1:A8H1Vmfg1BMEjKMhwJW0j6FRd/qTER+Je9JnXUfP7ks= -github.com/phrase/phrase-go/v3 v3.0.1/go.mod h1:8vb6fBPJ45NbvJ9rqP8+Auai+NjmgiHdGPMPFreI8Xw= +github.com/phrase/phrase-go/v3 v3.3.0 h1:kq2eFgKE6mUUZpud1KWsTa1RO4T+ztB2JI3DsCfqHog= +github.com/phrase/phrase-go/v3 v3.3.0/go.mod h1:s0uOYiXLxKAYlaIS6TbKv3efkKFUlY4OB6OL+VgvK90= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=