Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
be compatible for go 1.10
Browse files Browse the repository at this point in the history
bachue committed Apr 23, 2024
1 parent b6503e1 commit 22eacdc
Showing 21 changed files with 170 additions and 53 deletions.
3 changes: 3 additions & 0 deletions storagev2/apis/api_add_bucket_event_rule.go
Original file line number Diff line number Diff line change
@@ -82,6 +82,9 @@ func (storage *Storage) AddBucketEventRule(ctx context.Context, request *AddBuck
}
innerRequest := (*innerAddBucketEventRuleRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "events", "add")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_delete_bucket_event_rule.go
Original file line number Diff line number Diff line change
@@ -56,6 +56,9 @@ func (storage *Storage) DeleteBucketEventRule(ctx context.Context, request *Dele
}
innerRequest := (*innerDeleteBucketEventRuleRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "events", "delete")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_disable_bucket_index_page.go
Original file line number Diff line number Diff line change
@@ -53,6 +53,9 @@ func (storage *Storage) DisableBucketIndexPage(ctx context.Context, request *Dis
}
innerRequest := (*innerDisableBucketIndexPageRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "noIndexPage")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_get_bucket_domains_v3.go
Original file line number Diff line number Diff line change
@@ -51,6 +51,9 @@ func (storage *Storage) GetBucketDomainsV3(ctx context.Context, request *GetBuck
}
innerRequest := (*innerGetBucketDomainsV3Request)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "v3", "domains")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_get_bucket_event_rules.go
Original file line number Diff line number Diff line change
@@ -51,6 +51,9 @@ func (storage *Storage) GetBucketEventRules(ctx context.Context, request *GetBuc
}
innerRequest := (*innerGetBucketEventRulesRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "events", "get")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_get_bucket_info.go
Original file line number Diff line number Diff line change
@@ -51,6 +51,9 @@ func (storage *Storage) GetBucketInfo(ctx context.Context, request *GetBucketInf
}
innerRequest := (*innerGetBucketInfoRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "v2", "bucketInfo")
path := "/" + strings.Join(pathSegments, "/")
4 changes: 4 additions & 0 deletions storagev2/apis/api_get_bucket_infos.go
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ import (
"context"
auth "github.com/qiniu/go-sdk/v7/auth"
getbucketinfos "github.com/qiniu/go-sdk/v7/storagev2/apis/get_bucket_infos"
errors "github.com/qiniu/go-sdk/v7/storagev2/errors"
httpclient "github.com/qiniu/go-sdk/v7/storagev2/http_client"
uplog "github.com/qiniu/go-sdk/v7/storagev2/internal/uplog"
region "github.com/qiniu/go-sdk/v7/storagev2/region"
@@ -49,6 +50,9 @@ func (storage *Storage) GetBucketInfos(ctx context.Context, request *GetBucketIn
}
innerRequest := (*innerGetBucketInfosRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "v2", "bucketInfos")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_query_bucket_v2.go
Original file line number Diff line number Diff line change
@@ -56,6 +56,9 @@ func (storage *Storage) QueryBucketV2(ctx context.Context, request *QueryBucketV
}
innerRequest := (*innerQueryBucketV2Request)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "v2", "query")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_query_bucket_v4.go
Original file line number Diff line number Diff line change
@@ -56,6 +56,9 @@ func (storage *Storage) QueryBucketV4(ctx context.Context, request *QueryBucketV
}
innerRequest := (*innerQueryBucketV4Request)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "v4", "query")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_set_bucket_image.go
Original file line number Diff line number Diff line change
@@ -59,6 +59,9 @@ func (storage *Storage) SetBucketImage(ctx context.Context, request *SetBucketIm
}
innerRequest := (*innerSetBucketImageRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "image")
if segments, err := innerRequest.buildPath(); err != nil {
3 changes: 3 additions & 0 deletions storagev2/apis/api_set_bucket_refer_anti_leech.go
Original file line number Diff line number Diff line change
@@ -56,6 +56,9 @@ func (storage *Storage) SetBucketReferAntiLeech(ctx context.Context, request *Se
}
innerRequest := (*innerSetBucketReferAntiLeechRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "referAntiLeech")
path := "/" + strings.Join(pathSegments, "/")
3 changes: 3 additions & 0 deletions storagev2/apis/api_unset_bucket_image.go
Original file line number Diff line number Diff line change
@@ -50,6 +50,9 @@ func (storage *Storage) UnsetBucketImage(ctx context.Context, request *UnsetBuck
}
innerRequest := (*innerUnsetBucketImageRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "unimage")
if segments, err := innerRequest.buildPath(); err != nil {
3 changes: 3 additions & 0 deletions storagev2/apis/api_update_bucket_event_rule.go
Original file line number Diff line number Diff line change
@@ -82,6 +82,9 @@ func (storage *Storage) UpdateBucketEventRule(ctx context.Context, request *Upda
}
innerRequest := (*innerUpdateBucketEventRuleRequest)(request)
serviceNames := []region.ServiceName{region.ServiceBucket}
if innerRequest.Credentials == nil && storage.client.GetCredentials() == nil {
return nil, errors.MissingRequiredFieldError{Name: "Credentials"}
}
var pathSegments []string
pathSegments = append(pathSegments, "events", "update")
path := "/" + strings.Join(pathSegments, "/")
4 changes: 3 additions & 1 deletion storagev2/http_client/http_client.go
Original file line number Diff line number Diff line change
@@ -192,7 +192,9 @@ func NewClient(options *Options) *Client {
options.ShouldFreezeHost = defaultShouldFreezeHost
}
if options.Credentials == nil {
options.Credentials = auth.Default()
if defaultAuth := auth.Default(); defaultAuth != nil {
options.Credentials = defaultAuth
}
}

return &Client{
10 changes: 10 additions & 0 deletions storagev2/internal/uplog/dns1.12.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
//go:build !1.13
// +build !1.13

package uplog

import "net"

func isDnsNotFoundError(dnsError *net.DNSError) bool {
return !dnsError.IsTemporary
}
10 changes: 10 additions & 0 deletions storagev2/internal/uplog/dns1.13.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
//go:build 1.13
// +build 1.13

package uplog

import "net"

func isDnsNotFoundError(dnsError *net.DNSError) bool {
return dnsError.IsNotFound
}
62 changes: 38 additions & 24 deletions storagev2/internal/uplog/uplog.go
Original file line number Diff line number Diff line change
@@ -2,10 +2,7 @@ package uplog

import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/url"
"os"
"strings"
@@ -75,30 +72,47 @@ func getOsVersion() (string, error) {
}

func detectErrorType(err error) ErrorType {
var (
dnsError *net.DNSError
urlError *url.Error
tlsVerifyCertError *tls.CertificateVerificationError
syscallError syscall.Errno
)
if os.IsTimeout(err) || errors.Is(err, syscall.ETIMEDOUT) {
tryToUnwrapUnderlyingError := func(err error) (error, bool) {
switch err := err.(type) {
case *os.PathError:
return err.Err, true
case *os.LinkError:
return err.Err, true
case *os.SyscallError:
return err.Err, true
case *url.Error:
return err.Err, true
case *net.OpError:
return err.Err, true
}
return err, false
}
unwrapUnderlyingError := func(err error) error {
ok := true
for ok {
err, ok = tryToUnwrapUnderlyingError(err)
}
return err
}

unwrapedErr := unwrapUnderlyingError(err)
if os.IsTimeout(unwrapedErr) {
return ErrorTypeTimeout
} else if errors.As(err, &dnsError) && dnsError.IsNotFound {
} else if dnsError, ok := unwrapedErr.(*net.DNSError); ok && isDnsNotFoundError(dnsError) {
return ErrorTypeUnknownHost
} else if os.IsNotExist(err) || os.IsPermission(err) {
} else if os.IsNotExist(unwrapedErr) || os.IsPermission(unwrapedErr) {
return ErrorTypeLocalIoError
} else if errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNABORTED) || errors.Is(err, syscall.ECONNRESET) {
return ErrorTypeCannotConnectToHost
} else if errors.As(err, &syscallError) {
return ErrorTypeUnexpectedSyscallError
} else if errors.Is(err, context.Canceled) {
} else if syscallError, ok := unwrapedErr.(*os.SyscallError); ok {
switch syscallError.Err {
case syscall.ECONNREFUSED, syscall.ECONNABORTED, syscall.ECONNRESET:
return ErrorTypeCannotConnectToHost
default:
return ErrorTypeUnexpectedSyscallError
}
} else if unwrapedErr == context.Canceled {
return ErrorTypeUserCanceled
} else if errors.Is(err, http.ErrSchemeMismatch) {
return ErrorTypeProtocolError
} else if errors.As(err, &tlsVerifyCertError) {
return ErrorTypeSSLError
} else if errors.As(err, &urlError) {
desc := urlError.Err.Error()
} else {
desc := unwrapedErr.Error()
if strings.HasPrefix(desc, "tls: ") ||
strings.HasPrefix(desc, "x509: ") {
return ErrorTypeSSLError
@@ -108,8 +122,8 @@ func detectErrorType(err error) ErrorType {
strings.Contains(desc, "server closed idle connection") {
return ErrorTypeTransmissionError
}
return ErrorTypeUnknownError
}
return ErrorTypeUnknownError
}

func getHttpClientName() string {
11 changes: 7 additions & 4 deletions storagev2/internal/uplog/uplog_buffer.go
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import (
"bytes"
"compress/gzip"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
@@ -243,7 +244,8 @@ func SetWriteFileBufferInterval(d time.Duration) {
d = 1 * time.Minute
}
uplogWriteFileBufferInterval = d
uplogWriteFileBufferTicker.Reset(d)
uplogWriteFileBufferTicker.Stop()
uplogWriteFileBufferTicker = time.NewTicker(d)
}

func GetWriteFileBufferInterval() time.Duration {
@@ -256,7 +258,8 @@ func GetWriteFileBufferInterval() time.Duration {
func resetWriteFileBufferInterval() {
uplogWriteFileBufferTimerLock.Lock()
defer uplogWriteFileBufferTimerLock.Unlock()
uplogWriteFileBufferTicker.Reset(uplogWriteFileBufferInterval)
uplogWriteFileBufferTicker.Stop()
uplogWriteFileBufferTicker = time.NewTicker(uplogWriteFileBufferInterval)
}

type multipleFileReader struct {
@@ -330,14 +333,14 @@ func (r *multipleFileReader) Close() error {
}

func getArchivedUplogFileBufferPaths(dirPath string) ([]string, error) {
dirEntries, err := os.ReadDir(dirPath)
dirEntries, err := ioutil.ReadDir(dirPath)
if err != nil {
return nil, err
}

archivedPaths := make([]string, 0, len(dirEntries))
for _, dirEntry := range dirEntries {
if !dirEntry.Type().IsRegular() {
if !dirEntry.Mode().IsRegular() {
continue
}
if !strings.HasPrefix(dirEntry.Name(), UPLOG_FILE_BUFFER_NAME+".") {
10 changes: 10 additions & 0 deletions storagev2/retrier/dns1.12.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
//go:build !1.13
// +build !1.13

package retrier

import "net"

func isDnsNotFoundError(dnsError *net.DNSError) bool {
return !dnsError.IsTemporary
}
10 changes: 10 additions & 0 deletions storagev2/retrier/dns1.13.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
//go:build 1.13
// +build 1.13

package retrier

import "net"

func isDnsNotFoundError(dnsError *net.DNSError) bool {
return dnsError.IsNotFound
}
66 changes: 42 additions & 24 deletions storagev2/retrier/retrier.go
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@ package retrier

import (
"context"
"errors"
"net"
"net/http"
"net/url"
@@ -114,37 +113,56 @@ func getRetryDecisionForError(err error) RetryDecision {
return DontRetry
}

var dnsError *net.DNSError
if os.IsTimeout(err) || errors.Is(err, syscall.ETIMEDOUT) {
return RetryRequest
} else if errors.As(err, &dnsError) && dnsError.IsNotFound {
return TryNextHost
} else if errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNABORTED) || errors.Is(err, syscall.ECONNRESET) {
return TryNextHost
} else if errors.Is(err, context.Canceled) {
return DontRetry
} else if errors.Is(err, http.ErrSchemeMismatch) {
return DontRetry
tryToUnwrapUnderlyingError := func(err error) (error, bool) {
switch err := err.(type) {
case *os.PathError:
return err.Err, true
case *os.LinkError:
return err.Err, true
case *os.SyscallError:
return err.Err, true
case *url.Error:
return err.Err, true
case *net.OpError:
return err.Err, true
}
return err, false
}
unwrapUnderlyingError := func(err error) error {
ok := true
for ok {
err, ok = tryToUnwrapUnderlyingError(err)
}
return err
}

switch t := err.(type) {
case *url.Error:
desc := err.Error()
if strings.Contains(desc, "use of closed network connection") ||
strings.Contains(desc, "unexpected EOF reading trailer") ||
strings.Contains(desc, "transport connection broken") ||
strings.Contains(desc, "server closed idle connection") {
return RetryRequest
} else {
unwrapedErr := unwrapUnderlyingError(err)
if os.IsTimeout(unwrapedErr) {
return RetryRequest
} else if dnsError, ok := unwrapedErr.(*net.DNSError); ok && isDnsNotFoundError(dnsError) {
return TryNextHost
} else if syscallError, ok := unwrapedErr.(*os.SyscallError); ok {
switch syscallError.Err {
case syscall.ECONNREFUSED, syscall.ECONNABORTED, syscall.ECONNRESET:
return TryNextHost
default:
return DontRetry
}
case *clientv1.ErrorInfo:
if isStatusCodeRetryable(t.Code) {
} else if unwrapedErr == context.Canceled {
return DontRetry
} else if clientErr, ok := unwrapedErr.(*clientv1.ErrorInfo); ok {
if isStatusCodeRetryable(clientErr.Code) {
return RetryRequest
} else {
return DontRetry
}
default:
}
desc := unwrapedErr.Error()
if strings.Contains(desc, "use of closed network connection") ||
strings.Contains(desc, "unexpected EOF reading trailer") ||
strings.Contains(desc, "transport connection broken") ||
strings.Contains(desc, "server closed idle connection") {
return RetryRequest
}
return DontRetry
}

0 comments on commit 22eacdc

Please sign in to comment.