Skip to content

Commit 646c325

Browse files
jkowalskijkowalski
authored
Implemented new streaming GRPC protocol for Kopia Repository Server (kopia#789)
* grpcapi: added GPRC API for the repository server * repo: added transparent retries to GRPC repository client Normally GRPC reconnects automatically, which can survive server restarts (minus transient errors). In our case we're establishing a stream which will be broken and needs to be restarted after io.EOF is detected. It safe to do transparent retries for read-only (repo.Repository), but not safe for write sessions (repo.RepositoryWriter), because the session may re-connect to different server that won't have the buffered content write available in memory.
1 parent a11ddaf commit 646c325

24 files changed

+4059
-61
lines changed

‎Makefile‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ TEST_FLAGS?=
33
KOPIA_INTEGRATION_EXE=$(CURDIR)/dist/integration/kopia.exe
44
TESTING_ACTION_EXE=$(CURDIR)/dist/integration/testingaction.exe
55
FIO_DOCKER_TAG=ljishen/fio
6+
REPEAT_TEST=1
67

78
export BOTO_PATH=$(CURDIR)/tools/.boto
89

@@ -208,20 +209,20 @@ dev-deps:
208209
GO111MODULE=off go get -u github.com/newhook/go-symbols
209210
GO111MODULE=off go get -u github.com/sqs/goreturns
210211

211-
212212
test-with-coverage: export RCLONE_EXE=$(rclone)
213-
test-with-coverage: $(rclone)
214-
$(GO_TEST) -count=1 -coverprofile=tmp.cov --coverpkg $(COVERAGE_PACKAGES) -timeout 300s $(shell go list ./...)
213+
test-with-coverage:
214+
$(GO_TEST) -count=$(REPEAT_TEST) -coverprofile=tmp.cov --coverpkg $(COVERAGE_PACKAGES) -timeout 300s $(shell go list ./...)
215215

216+
test-with-coverage-pkgonly: export RCLONE_EXE=$(rclone)
216217
test-with-coverage-pkgonly:
217-
$(GO_TEST) -count=1 -coverprofile=tmp.cov -timeout 300s github.com/kopia/kopia/...
218+
$(GO_TEST) -count=$(REPEAT_TEST) -coverprofile=tmp.cov -timeout 300s github.com/kopia/kopia/...
218219

219220
test: export RCLONE_EXE=$(rclone)
220-
test: $(gotestsum) $(rclone)
221-
$(GO_TEST) -count=1 -timeout $(UNIT_TESTS_TIMEOUT) ./...
221+
test: $(gotestsum)
222+
$(GO_TEST) -count=$(REPEAT_TEST) -timeout $(UNIT_TESTS_TIMEOUT) ./...
222223

223224
vtest: $(gotestsum)
224-
$(GO_TEST) -count=1 -short -v -timeout $(UNIT_TESTS_TIMEOUT) ./...
225+
$(GO_TEST) -count=$(REPEAT_TEST) -short -v -timeout $(UNIT_TESTS_TIMEOUT) ./...
225226

226227
build-integration-test-binary:
227228
go build -o $(KOPIA_INTEGRATION_EXE) -tags testing github.com/kopia/kopia
@@ -232,26 +233,26 @@ $(TESTING_ACTION_EXE): tests/testingaction/main.go
232233
integration-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
233234
integration-tests: export TESTING_ACTION_EXE ?= $(TESTING_ACTION_EXE)
234235
integration-tests: build-integration-test-binary $(gotestsum) $(TESTING_ACTION_EXE)
235-
$(GO_TEST) $(TEST_FLAGS) -count=1 -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/end_to_end_test
236+
$(GO_TEST) $(TEST_FLAGS) -count=$(REPEAT_TEST) -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/end_to_end_test
236237

237238
endurance-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
238239
endurance-tests: build-integration-test-binary $(gotestsum)
239-
$(GO_TEST) $(TEST_FLAGS) -count=1 -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/endurance_test
240+
$(GO_TEST) $(TEST_FLAGS) -count=$(REPEAT_TEST) -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/endurance_test
240241

241242
robustness-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
242243
robustness-tests: build-integration-test-binary $(gotestsum)
243244
FIO_DOCKER_IMAGE=$(FIO_DOCKER_TAG) \
244-
$(GO_TEST) -count=1 github.com/kopia/kopia/tests/robustness/robustness_test $(TEST_FLAGS)
245+
$(GO_TEST) -count=$(REPEAT_TEST) github.com/kopia/kopia/tests/robustness/robustness_test $(TEST_FLAGS)
245246

246247
robustness-tool-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
247248
robustness-tool-tests: build-integration-test-binary $(gotestsum)
248249
KOPIA_EXE=$(KOPIA_INTEGRATION_EXE) \
249250
FIO_DOCKER_IMAGE=$(FIO_DOCKER_TAG) \
250-
$(GO_TEST) -count=1 github.com/kopia/kopia/tests/tools/... github.com/kopia/kopia/tests/robustness/engine/... $(TEST_FLAGS)
251+
$(GO_TEST) -count=$(REPEAT_TEST) github.com/kopia/kopia/tests/tools/... github.com/kopia/kopia/tests/robustness/engine/... $(TEST_FLAGS)
251252

252253
stress-test: $(gotestsum)
253-
KOPIA_LONG_STRESS_TEST=1 $(GO_TEST) -count=1 -timeout 200s github.com/kopia/kopia/tests/stress_test
254-
$(GO_TEST) -count=1 -timeout 200s github.com/kopia/kopia/tests/repository_stress_test
254+
KOPIA_LONG_STRESS_TEST=1 $(GO_TEST) -count=$(REPEAT_TEST) -timeout 200s github.com/kopia/kopia/tests/stress_test
255+
$(GO_TEST) -count=$(REPEAT_TEST) -timeout 200s github.com/kopia/kopia/tests/repository_stress_test
255256

256257
layering-test:
257258
ifneq ($(uname),Windows)

‎cli/command_repository_connect_server.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ var (
1414

1515
connectAPIServerURL = connectAPIServerCommand.Flag("url", "Server URL").Required().String()
1616
connectAPIServerCertFingerprint = connectAPIServerCommand.Flag("server-cert-fingerprint", "Server certificate fingerprint").String()
17+
connectAPIServerUseGRPCAPI = connectAPIServerCommand.Flag("grpc", "Use GRPC API").Default("true").Bool()
1718
)
1819

1920
func runConnectAPIServerCommand(ctx context.Context) error {
@@ -25,6 +26,7 @@ func runConnectAPIServerCommand(ctx context.Context) error {
2526
as := &repo.APIServerInfo{
2627
BaseURL: strings.TrimSuffix(*connectAPIServerURL, "/"),
2728
TrustedServerCertificateFingerprint: strings.ToLower(*connectAPIServerCertFingerprint),
29+
DisableGRPC: !*connectAPIServerUseGRPCAPI,
2830
}
2931

3032
configFile := repositoryConfigFileName()

‎cli/command_server_start.go‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/pkg/errors"
1919
prom "github.com/prometheus/client_golang/prometheus"
2020
htpasswd "github.com/tg123/go-htpasswd"
21+
"google.golang.org/grpc"
2122

2223
"github.com/kopia/kopia/internal/auth"
2324
"github.com/kopia/kopia/internal/clock"
@@ -26,10 +27,15 @@ import (
2627
)
2728

2829
var (
29-
serverStartCommand = serverCommands.Command("start", "Start Kopia server").Default()
30-
serverStartHTMLPath = serverStartCommand.Flag("html", "Server the provided HTML at the root URL").ExistingDir()
31-
serverStartUI = serverStartCommand.Flag("ui", "Start the server with HTML UI").Default("true").Bool()
32-
serverStartRefreshInterval = serverStartCommand.Flag("refresh-interval", "Frequency for refreshing repository status").Default("10s").Duration()
30+
serverStartCommand = serverCommands.Command("start", "Start Kopia server").Default()
31+
serverStartHTMLPath = serverStartCommand.Flag("html", "Server the provided HTML at the root URL").ExistingDir()
32+
serverStartUI = serverStartCommand.Flag("ui", "Start the server with HTML UI").Default("true").Bool()
33+
34+
serverStartLegacyRepositoryAPI = serverStartCommand.Flag("legacy-api", "Start the legacy server API").Default("true").Bool()
35+
serverStartGRPC = serverStartCommand.Flag("grpc", "Start the GRPC server").Default("true").Bool()
36+
37+
serverStartRefreshInterval = serverStartCommand.Flag("refresh-interval", "Frequency for refreshing repository status").Default("300s").Duration()
38+
serverStartMaxConcurrency = serverStartCommand.Flag("max-concurrency", "Maximum number of server goroutines").Default("0").Int()
3339

3440
serverStartRandomPassword = serverStartCommand.Flag("random-password", "Generate random password and print to stderr").Hidden().Bool()
3541
serverStartAutoShutdown = serverStartCommand.Flag("auto-shutdown", "Auto shutdown the server if API requests not received within given time").Hidden().Duration()
@@ -54,6 +60,7 @@ func runServer(ctx context.Context, rep repo.Repository) error {
5460
ConfigFile: repositoryConfigFileName(),
5561
ConnectOptions: connectOptions(),
5662
RefreshInterval: *serverStartRefreshInterval,
63+
MaxConcurrency: *serverStartMaxConcurrency,
5764
Authenticator: authn,
5865
Authorizer: auth.LegacyAuthorizerForUser,
5966
})
@@ -71,7 +78,7 @@ func runServer(ctx context.Context, rep repo.Repository) error {
7178

7279
mux := http.NewServeMux()
7380

74-
mux.Handle("/api/", srv.APIHandlers())
81+
mux.Handle("/api/", srv.APIHandlers(*serverStartLegacyRepositoryAPI))
7582

7683
if *serverStartHTMLPath != "" {
7784
fileServer := serveIndexFileForKnownUIRoutes(http.Dir(*serverStartHTMLPath))
@@ -108,7 +115,26 @@ func runServer(ctx context.Context, rep repo.Repository) error {
108115
})
109116
}
110117

111-
httpServer.Handler = handler
118+
if *serverStartGRPC {
119+
grpcServer := grpc.NewServer(
120+
grpc.MaxSendMsgSize(repo.MaxGRPCMessageSize),
121+
grpc.MaxRecvMsgSize(repo.MaxGRPCMessageSize),
122+
)
123+
srv.RegisterGRPCHandlers(grpcServer)
124+
125+
log(ctx).Debugf("starting GRPC/HTTP server...")
126+
127+
httpServer.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
128+
if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") {
129+
grpcServer.ServeHTTP(w, r)
130+
} else {
131+
handler.ServeHTTP(w, r)
132+
}
133+
})
134+
} else {
135+
log(ctx).Debugf("starting HTTP-only server...")
136+
httpServer.Handler = handler
137+
}
112138

113139
err = startServerWithOptionalTLS(ctx, httpServer)
114140
if !errors.Is(err, http.ErrServerClosed) {

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ require (
5252
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
5353
golang.org/x/sys v0.0.0-20200922070232-aee5d888a860
5454
google.golang.org/api v0.32.0
55+
google.golang.org/grpc v1.32.0
5556
google.golang.org/protobuf v1.25.0
5657
gopkg.in/kothar/go-backblaze.v0 v0.0.0-20191215213626-7594ed38700f
5758
)

‎internal/grpcapi/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
rebuild:
2+
protoc --go_out=. --go_opt=paths=source_relative \
3+
--go-grpc_out=. --go-grpc_opt=paths=source_relative \
4+
repository_server.proto

0 commit comments

Comments
 (0)