diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index a09886b..fee6cf9 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -32,19 +32,19 @@ jobs: run: python -m pip install --upgrade pip setuptools wheel - name: Install Python connector requirements - working-directory: v1/examples/source_connector/python + working-directory: examples/source_connector/python run: pip install -r requirements.txt - name: Run Python connector build script - working-directory: v1/examples/source_connector/python + working-directory: examples/source_connector/python run: ./build.sh - name: Install Python destination requirements - working-directory: v1/examples/destination_connector/python + working-directory: examples/destination_connector/python run: pip install -r requirements.txt - name: Run Python destination build script - working-directory: v1/examples/destination_connector/python + working-directory: examples/destination_connector/python run: ./build.sh - name: Set up Go @@ -58,45 +58,45 @@ jobs: version: "23.2" - name: Set up protobuf for Go - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: go install google.golang.org/protobuf/cmd/protoc-gen-go@latest - name: Set up protoc-gen-go - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest - name: Set up Go protoc path - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: export PATH="$PATH:$(go env GOPATH)/bin" - name: Verify dependencies - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: go mod verify - name: Build Go connector - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: scripts/build.sh - name: Run go vet - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: go vet ./... - name: Install staticcheck run: go install honnef.co/go/tools/cmd/staticcheck@latest - name: Run staticcheck - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: staticcheck ./... - name: Install golint run: go install golang.org/x/lint/golint@latest - name: Run golint - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: golint ./... - name: Run tests - working-directory: v1/examples/source_connector/golang + working-directory: examples/source_connector/golang run: go test -race -vet=off ./... - name: Setup Gradle @@ -105,17 +105,17 @@ jobs: gradle-version: 8.5 - name: Run Java connector copyProtos - working-directory: v1/examples/source_connector/java + working-directory: examples/source_connector/java run: gradle copyProtos - name: Run Java connector test with Gradle Wrapper - working-directory: v1/examples/source_connector/java + working-directory: examples/source_connector/java run: gradle build - name: Run Java destination copyProtos - working-directory: v1/examples/destination_connector/java + working-directory: examples/destination_connector/java run: gradle copyProtos - name: Run Java destination test with Gradle Wrapper - working-directory: v1/examples/destination_connector/java - run: gradle build \ No newline at end of file + working-directory: examples/destination_connector/java + run: gradle build diff --git a/.gitignore b/.gitignore index b390fc9..d625fd5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,20 +5,13 @@ **/build/ **/golang/main bin/* -v1/*/**/*.proto -v1/**/*.pb.go -v1/**/*pb2.py -v1/**/*pb2.pyi -v1/**/*pb2_grpc.py destination_run/ connector_run/ **/__pycache__/ -v1/**/node_modules/ -v1/**/package-lock.json -v2/*/**/*.proto -v2/**/*.pb.go -v2/**/*pb2.py -v2/**/*pb2.pyi -v2/**/*pb2_grpc.py -v2/**/node_modules/ -v2/**/package-lock.json \ No newline at end of file +examples/**/*.proto +examples/**/*.pb.go +examples/**/*pb2.py +examples/**/*pb2.pyi +examples/**/*pb2_grpc.py +examples/**/node_modules/ +examples/**/package-lock.json \ No newline at end of file diff --git a/README.md b/README.md index 4663581..c9f51bf 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,13 @@ A way for partners to create source and destination connectors that run on [Five ## Repo Structure This repo consists of example source and destination connectors along with a local testing environment. Both connectors are written in a [gRPC supported language](https://grpc.io/docs/languages/) that can generate a statically linked binary. We recommend Java, Golang, or Rust. -* [V2 Examples](v2/examples) -* [V1 Examples](v1/examples) +* [V2 Examples](examples) * [Local Testing Tools](tools/) ## Development You can follow the [SDK Development Guide](development-guide.md) for guidance on how to develop your code. -> NOTE: If you're a new partner that just started working with the Connector SDK, use [V2 protos](v2). We will continue to support [V1 protos](v1) for partners that have used them to build their SDK connectors, and these will be removed once the migration to V2 is complete. +> NOTE: If you're a new partner that just started working with the Connector SDK, use V2 release. We will continue to support V1 release for partners that have used them to build their SDK connectors, and these will be removed once the migration to V2 is complete. Once you have your code ready to run: 1. Start up your connector running on port 50051 (for destination code, use port 50052). diff --git a/v2/common_v2.proto b/common.proto similarity index 100% rename from v2/common_v2.proto rename to common.proto diff --git a/v1/connector_sdk.proto b/connector_sdk.proto similarity index 81% rename from v1/connector_sdk.proto rename to connector_sdk.proto index 4d906ee..d4d08d3 100644 --- a/v1/connector_sdk.proto +++ b/connector_sdk.proto @@ -1,13 +1,13 @@ syntax = "proto3"; option optimize_for = SPEED; option java_multiple_files = true; -option go_package = "fivetran.com/fivetran_sdk"; -package fivetran_sdk; +option go_package = "fivetran.com/fivetran_sdk_v2"; +package fivetran_sdk.v2; import "common.proto"; -// Fivetran (grpc client) <> Connector (grpc server) -service Connector { +// Fivetran (grpc client) <> SourceConnector (grpc server) +service SourceConnector { rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} rpc Test (TestRequest) returns (TestResponse) {} rpc Schema (SchemaRequest) returns (SchemaResponse) {} @@ -65,28 +65,12 @@ message TableSelection { } message UpdateResponse { - oneof response { - LogEntry log_entry = 1; - Operation operation = 2; - } -} - -enum LogLevel { - INFO = 0; - WARNING = 1; - SEVERE = 2; -} - -message LogEntry { - LogLevel level = 1; - string message = 2; -} - -message Operation { - oneof op { + oneof operation { Record record = 1; SchemaChange schema_change = 2; Checkpoint checkpoint = 3; + Warning warning = 4; + Task task = 5; } } @@ -100,10 +84,10 @@ message SchemaChange { message Record { optional string schema_name = 1; string table_name = 2; - OpType type = 3; + RecordType type = 3; map data = 4; } message Checkpoint { string state_json = 1; -} \ No newline at end of file +} diff --git a/v2/destination_connector_sdk_v2.proto b/destination_sdk.proto similarity index 99% rename from v2/destination_connector_sdk_v2.proto rename to destination_sdk.proto index e87848a..9ace90a 100644 --- a/v2/destination_connector_sdk_v2.proto +++ b/destination_sdk.proto @@ -5,7 +5,7 @@ option go_package = "fivetran.com/fivetran_sdk_v2"; package fivetran_sdk.v2; import "google/protobuf/timestamp.proto"; -import "common_v2.proto"; +import "common.proto"; // Fivetran (grpc client) <> DestinationConnector (grpc server) service DestinationConnector { diff --git a/development-guide.md b/development-guide.md index 26fc546..b172b69 100644 --- a/development-guide.md +++ b/development-guide.md @@ -21,7 +21,7 @@ The executable needs to do the following: * Partners should not add the proto files to their repos. Proto files should be pulled in from this repo at build time and added to `.gitignore` so they are excluded. * Always use proto files from latest release and update you're code if necessary. Older releases proto files can be considered deprecated and will be expired at later date. -* New partners that just started working with the Connector SDK should use [V2 protos](v2). Only partners that have already used [V1 protos](v1) to build SDK connectors should continue using them. +* New partners that just started working with the Connector SDK should use V2 release. We will continue to support V1 release for partners that have used them to build their SDK connectors, and these will be removed once the migration to V2 is complete. ### Logging diff --git a/v1/examples/destination_connector/java/.gitattributes b/examples/destination_connector/java/.gitattributes similarity index 100% rename from v1/examples/destination_connector/java/.gitattributes rename to examples/destination_connector/java/.gitattributes diff --git a/v1/examples/destination_connector/java/Dockerfile b/examples/destination_connector/java/Dockerfile similarity index 100% rename from v1/examples/destination_connector/java/Dockerfile rename to examples/destination_connector/java/Dockerfile diff --git a/v1/examples/destination_connector/java/README.md b/examples/destination_connector/java/README.md similarity index 100% rename from v1/examples/destination_connector/java/README.md rename to examples/destination_connector/java/README.md diff --git a/v1/examples/destination_connector/java/build.gradle b/examples/destination_connector/java/build.gradle similarity index 100% rename from v1/examples/destination_connector/java/build.gradle rename to examples/destination_connector/java/build.gradle diff --git a/v1/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.jar b/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from v1/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.jar rename to examples/destination_connector/java/gradle/wrapper/gradle-wrapper.jar diff --git a/v1/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.properties b/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from v1/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.properties rename to examples/destination_connector/java/gradle/wrapper/gradle-wrapper.properties diff --git a/v1/examples/destination_connector/java/gradlew b/examples/destination_connector/java/gradlew similarity index 100% rename from v1/examples/destination_connector/java/gradlew rename to examples/destination_connector/java/gradlew diff --git a/v1/examples/destination_connector/java/gradlew.bat b/examples/destination_connector/java/gradlew.bat similarity index 100% rename from v1/examples/destination_connector/java/gradlew.bat rename to examples/destination_connector/java/gradlew.bat diff --git a/v1/examples/destination_connector/java/settings.gradle b/examples/destination_connector/java/settings.gradle similarity index 100% rename from v1/examples/destination_connector/java/settings.gradle rename to examples/destination_connector/java/settings.gradle diff --git a/v2/examples/destination_connector/java/src/main/java/destination/DestinationServiceImpl.java b/examples/destination_connector/java/src/main/java/destination/DestinationServiceImpl.java similarity index 100% rename from v2/examples/destination_connector/java/src/main/java/destination/DestinationServiceImpl.java rename to examples/destination_connector/java/src/main/java/destination/DestinationServiceImpl.java diff --git a/v1/examples/destination_connector/java/src/main/java/destination/JavaDestination.java b/examples/destination_connector/java/src/main/java/destination/JavaDestination.java similarity index 100% rename from v1/examples/destination_connector/java/src/main/java/destination/JavaDestination.java rename to examples/destination_connector/java/src/main/java/destination/JavaDestination.java diff --git a/v1/examples/destination_connector/python/README.md b/examples/destination_connector/python/README.md similarity index 100% rename from v1/examples/destination_connector/python/README.md rename to examples/destination_connector/python/README.md diff --git a/v1/examples/destination_connector/python/build.sh b/examples/destination_connector/python/build.sh similarity index 100% rename from v1/examples/destination_connector/python/build.sh rename to examples/destination_connector/python/build.sh diff --git a/v2/examples/destination_connector/python/main.py b/examples/destination_connector/python/main.py similarity index 100% rename from v2/examples/destination_connector/python/main.py rename to examples/destination_connector/python/main.py diff --git a/v1/examples/destination_connector/python/read_csv.py b/examples/destination_connector/python/read_csv.py similarity index 100% rename from v1/examples/destination_connector/python/read_csv.py rename to examples/destination_connector/python/read_csv.py diff --git a/v1/examples/destination_connector/python/requirements.txt b/examples/destination_connector/python/requirements.txt similarity index 100% rename from v1/examples/destination_connector/python/requirements.txt rename to examples/destination_connector/python/requirements.txt diff --git a/v1/examples/destination_connector/python/run.sh b/examples/destination_connector/python/run.sh similarity index 100% rename from v1/examples/destination_connector/python/run.sh rename to examples/destination_connector/python/run.sh diff --git a/v1/examples/source_connector/golang/Dockerfile b/examples/source_connector/golang/Dockerfile similarity index 100% rename from v1/examples/source_connector/golang/Dockerfile rename to examples/source_connector/golang/Dockerfile diff --git a/v2/examples/source_connector/golang/README.md b/examples/source_connector/golang/README.md similarity index 100% rename from v2/examples/source_connector/golang/README.md rename to examples/source_connector/golang/README.md diff --git a/v1/examples/source_connector/golang/go.mod b/examples/source_connector/golang/go.mod similarity index 100% rename from v1/examples/source_connector/golang/go.mod rename to examples/source_connector/golang/go.mod diff --git a/v1/examples/source_connector/golang/go.sum b/examples/source_connector/golang/go.sum similarity index 100% rename from v1/examples/source_connector/golang/go.sum rename to examples/source_connector/golang/go.sum diff --git a/v2/examples/source_connector/golang/golang_connector/main.go b/examples/source_connector/golang/golang_connector/main.go similarity index 100% rename from v2/examples/source_connector/golang/golang_connector/main.go rename to examples/source_connector/golang/golang_connector/main.go diff --git a/v1/examples/source_connector/golang/scripts/build.sh b/examples/source_connector/golang/scripts/build.sh similarity index 100% rename from v1/examples/source_connector/golang/scripts/build.sh rename to examples/source_connector/golang/scripts/build.sh diff --git a/v1/examples/source_connector/golang/scripts/compile_protos.sh b/examples/source_connector/golang/scripts/compile_protos.sh similarity index 100% rename from v1/examples/source_connector/golang/scripts/compile_protos.sh rename to examples/source_connector/golang/scripts/compile_protos.sh diff --git a/v1/examples/source_connector/golang/scripts/copy_protos.sh b/examples/source_connector/golang/scripts/copy_protos.sh similarity index 100% rename from v1/examples/source_connector/golang/scripts/copy_protos.sh rename to examples/source_connector/golang/scripts/copy_protos.sh diff --git a/v1/examples/source_connector/golang/scripts/run.sh b/examples/source_connector/golang/scripts/run.sh similarity index 100% rename from v1/examples/source_connector/golang/scripts/run.sh rename to examples/source_connector/golang/scripts/run.sh diff --git a/v1/examples/source_connector/java/.gitattributes b/examples/source_connector/java/.gitattributes similarity index 100% rename from v1/examples/source_connector/java/.gitattributes rename to examples/source_connector/java/.gitattributes diff --git a/v1/examples/source_connector/java/Dockerfile b/examples/source_connector/java/Dockerfile similarity index 100% rename from v1/examples/source_connector/java/Dockerfile rename to examples/source_connector/java/Dockerfile diff --git a/v1/examples/source_connector/java/README.md b/examples/source_connector/java/README.md similarity index 100% rename from v1/examples/source_connector/java/README.md rename to examples/source_connector/java/README.md diff --git a/v1/examples/source_connector/java/build.gradle b/examples/source_connector/java/build.gradle similarity index 100% rename from v1/examples/source_connector/java/build.gradle rename to examples/source_connector/java/build.gradle diff --git a/v1/examples/source_connector/java/gradle/wrapper/gradle-wrapper.jar b/examples/source_connector/java/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from v1/examples/source_connector/java/gradle/wrapper/gradle-wrapper.jar rename to examples/source_connector/java/gradle/wrapper/gradle-wrapper.jar diff --git a/v1/examples/source_connector/java/gradle/wrapper/gradle-wrapper.properties b/examples/source_connector/java/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from v1/examples/source_connector/java/gradle/wrapper/gradle-wrapper.properties rename to examples/source_connector/java/gradle/wrapper/gradle-wrapper.properties diff --git a/v1/examples/source_connector/java/gradlew b/examples/source_connector/java/gradlew similarity index 100% rename from v1/examples/source_connector/java/gradlew rename to examples/source_connector/java/gradlew diff --git a/v1/examples/source_connector/java/gradlew.bat b/examples/source_connector/java/gradlew.bat similarity index 100% rename from v1/examples/source_connector/java/gradlew.bat rename to examples/source_connector/java/gradlew.bat diff --git a/v1/examples/source_connector/java/settings.gradle b/examples/source_connector/java/settings.gradle similarity index 100% rename from v1/examples/source_connector/java/settings.gradle rename to examples/source_connector/java/settings.gradle diff --git a/v2/examples/source_connector/java/src/main/java/connector/ConnectorServiceImpl.java b/examples/source_connector/java/src/main/java/connector/ConnectorServiceImpl.java similarity index 100% rename from v2/examples/source_connector/java/src/main/java/connector/ConnectorServiceImpl.java rename to examples/source_connector/java/src/main/java/connector/ConnectorServiceImpl.java diff --git a/v1/examples/source_connector/java/src/main/java/connector/JavaConnector.java b/examples/source_connector/java/src/main/java/connector/JavaConnector.java similarity index 100% rename from v1/examples/source_connector/java/src/main/java/connector/JavaConnector.java rename to examples/source_connector/java/src/main/java/connector/JavaConnector.java diff --git a/v1/examples/source_connector/java/src/main/java/connector/State.java b/examples/source_connector/java/src/main/java/connector/State.java similarity index 100% rename from v1/examples/source_connector/java/src/main/java/connector/State.java rename to examples/source_connector/java/src/main/java/connector/State.java diff --git a/v1/examples/source_connector/nodejs/README.md b/examples/source_connector/nodejs/README.md similarity index 100% rename from v1/examples/source_connector/nodejs/README.md rename to examples/source_connector/nodejs/README.md diff --git a/v1/examples/source_connector/nodejs/build.sh b/examples/source_connector/nodejs/build.sh similarity index 100% rename from v1/examples/source_connector/nodejs/build.sh rename to examples/source_connector/nodejs/build.sh diff --git a/v1/examples/source_connector/nodejs/package.json b/examples/source_connector/nodejs/package.json similarity index 100% rename from v1/examples/source_connector/nodejs/package.json rename to examples/source_connector/nodejs/package.json diff --git a/v2/examples/source_connector/nodejs/src/index.js b/examples/source_connector/nodejs/src/index.js similarity index 98% rename from v2/examples/source_connector/nodejs/src/index.js rename to examples/source_connector/nodejs/src/index.js index 30c79b3..37222db 100644 --- a/v2/examples/source_connector/nodejs/src/index.js +++ b/examples/source_connector/nodejs/src/index.js @@ -1,5 +1,5 @@ const grpc = require("@grpc/grpc-js"); -const PROTO_PATH_CONNECTOR = "./src/protos/source_connector_sdk_v2.proto"; +const PROTO_PATH_CONNECTOR = "./src/protos/connector_sdk.proto"; var protoLoader = require("@grpc/proto-loader"); const options = { diff --git a/v1/examples/source_connector/python/README.md b/examples/source_connector/python/README.md similarity index 100% rename from v1/examples/source_connector/python/README.md rename to examples/source_connector/python/README.md diff --git a/v1/examples/source_connector/python/build.sh b/examples/source_connector/python/build.sh similarity index 100% rename from v1/examples/source_connector/python/build.sh rename to examples/source_connector/python/build.sh diff --git a/v2/examples/source_connector/python/main.py b/examples/source_connector/python/main.py similarity index 99% rename from v2/examples/source_connector/python/main.py rename to examples/source_connector/python/main.py index 1d537f8..74d17f6 100644 --- a/v2/examples/source_connector/python/main.py +++ b/examples/source_connector/python/main.py @@ -1,5 +1,3 @@ -from sys import api_version - import grpc from concurrent import futures import json diff --git a/v1/examples/source_connector/python/requirements.txt b/examples/source_connector/python/requirements.txt similarity index 100% rename from v1/examples/source_connector/python/requirements.txt rename to examples/source_connector/python/requirements.txt diff --git a/v1/examples/source_connector/python/run.sh b/examples/source_connector/python/run.sh similarity index 100% rename from v1/examples/source_connector/python/run.sh rename to examples/source_connector/python/run.sh diff --git a/tools/destination-connector-tester/README.md b/tools/destination-connector-tester/README.md index 56dd0ab..c319e5b 100644 --- a/tools/destination-connector-tester/README.md +++ b/tools/destination-connector-tester/README.md @@ -2,7 +2,7 @@ ## Pre-requisites - Docker Desktop >= 4.23.0 or [Rancher Desktop](https://rancherdesktop.io/) >= 1.12.1 -- gRPC server is running for the particular example (see [example readme's](/v2/examples/destination_connector/)) +- gRPC server is running for the particular example (see [example readme's](/examples/destination_connector/)) ## How To Run @@ -17,8 +17,6 @@ docker pull us-docker.pkg.dev/build-286712/public-docker-us/sdktesters-v2/sdk-tester: ``` -> NOTE: If using V1 proto versions, use the latest docker image of the [public-docker-us/sdktesters/sdk-tester](https://console.cloud.google.com/artifacts/browse/build-286712/us/public-docker-us/sdktesters%2Fsdk-tester) artifact in Google Artifact Registry. - 2. Run a container using the image with the following command. Make sure to map a local directory for the tool by replacing `` placeholders in the command, and replace `` with the version of the image you pulled. ``` diff --git a/tools/source-connector-tester/README.md b/tools/source-connector-tester/README.md index 55b58ac..8498627 100644 --- a/tools/source-connector-tester/README.md +++ b/tools/source-connector-tester/README.md @@ -2,7 +2,7 @@ ## Pre-requisites - Docker Desktop >= 4.23.0 or [Rancher Desktop](https://rancherdesktop.io/) >= 1.12.1 -- gRPC server is running for the particular example (see [example readme's](/v2/examples/source_connector/)) +- gRPC server is running for the particular example (see [example readme's](/examples/source_connector/)) ## How To Run @@ -17,8 +17,7 @@ ``` docker pull us-docker.pkg.dev/build-286712/public-docker-us/sdktesters-v2/sdk-tester: ``` -> NOTE: If using V1 proto versions, use the latest docker image of the [public-docker-us/sdktesters/sdk-tester](https://console.cloud.google.com/artifacts/browse/build-286712/us/public-docker-us/sdktesters%2Fsdk-tester) artifact in Google Artifact Registry. - + 2. Run a container using the image with the following command. Make sure to map a local directory for storing files that the tool generates by replacing `` in the command, and replace with the version of the image you pulled. ``` diff --git a/v1/common.proto b/v1/common.proto deleted file mode 100644 index 2f09d03..0000000 --- a/v1/common.proto +++ /dev/null @@ -1,133 +0,0 @@ -syntax = "proto3"; -option optimize_for = SPEED; -option java_multiple_files = true; -option go_package = "fivetran.com/fivetran_sdk"; -package fivetran_sdk; - -import "google/protobuf/timestamp.proto"; - -message ConfigurationFormRequest {} - -message ConfigurationFormResponse { - bool schema_selection_supported = 1; - bool table_selection_supported = 2; - repeated FormField fields = 3; - repeated ConfigurationTest tests = 4; -} - -message FormField { - string name = 1; - string label = 2; - bool required = 3; - optional string description = 4; - oneof type { - TextField text_field = 5; - DropdownField dropdown_field = 6; - ToggleField toggle_field = 7; - } -} - -message DropdownField { - repeated string dropdown_field = 1; -} - -message ToggleField {} - -enum TextField { - PlainText = 0; - Password = 1; - Hidden = 2; -} - -message ConfigurationTest { - string name = 1; // unique identifier for the test - string label = 2; // A few words indicating what we are testing, e.g. 'Connecting to database' -} - -message TestRequest { - string name = 1; - map configuration = 2; -} - -message TestResponse { - oneof response { - bool success = 1; - string failure = 2; - // potential future warning - } -} - -message SchemaList { - repeated Schema schemas = 1; -} - -message TableList { - repeated Table tables = 1; -} - -message Schema { - string name = 1; - repeated Table tables = 2; -} - -enum DataType { - UNSPECIFIED = 0; - BOOLEAN = 1; - SHORT = 2; - INT = 3; - LONG = 4; - DECIMAL = 5; - FLOAT = 6; - DOUBLE = 7; - NAIVE_DATE = 8; - NAIVE_DATETIME = 9; - UTC_DATETIME = 10; - BINARY = 11; - XML = 12; - STRING = 13; - JSON = 14; -} - -message DecimalParams { - uint32 precision = 1; - uint32 scale = 2; -} - -enum OpType { - UPSERT = 0; - UPDATE = 1; - DELETE = 2; - TRUNCATE = 3; -} - -message ValueType { - oneof inner { - bool null = 1; - bool bool = 2; - int32 short = 3; - int32 int = 4; - int64 long = 5; - float float = 6; - double double = 7; - google.protobuf.Timestamp naive_date = 8; - google.protobuf.Timestamp naive_datetime = 9; - google.protobuf.Timestamp utc_datetime = 10; - string decimal = 11; - bytes binary = 12; - string string = 13; - string json = 14; - string xml = 15; - } -} - -message Table { - string name = 1; - repeated Column columns = 2; -} - -message Column { - string name = 1; - DataType type = 2; - bool primary_key = 3; - optional DecimalParams decimal = 4; -} diff --git a/v1/destination_sdk.proto b/v1/destination_sdk.proto deleted file mode 100644 index 8c5635b..0000000 --- a/v1/destination_sdk.proto +++ /dev/null @@ -1,134 +0,0 @@ -syntax = "proto3"; -option optimize_for = SPEED; -option java_multiple_files = true; -option go_package = "fivetran.com/fivetran_sdk"; -package fivetran_sdk; - -import "google/protobuf/timestamp.proto"; -import "common.proto"; - -// Fivetran (grpc client) <> Destination (grpc server) -service Destination { - rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} - rpc Capabilities (CapabilitiesRequest) returns (CapabilitiesResponse) {} - rpc Test (TestRequest) returns (TestResponse) {} - rpc DescribeTable (DescribeTableRequest) returns (DescribeTableResponse) {} - rpc CreateTable(CreateTableRequest) returns (CreateTableResponse) {} - rpc AlterTable(AlterTableRequest) returns (AlterTableResponse) {} - rpc Truncate(TruncateRequest) returns (TruncateResponse) {} - rpc WriteBatch (WriteBatchRequest) returns (WriteBatchResponse) {} -} - -message CapabilitiesRequest {} - -message CapabilitiesResponse { - BatchFileFormat batch_file_format = 1; -} - -message DescribeTableRequest { - map configuration = 1; - string schema_name = 2; - string table_name = 3; -} - -message DescribeTableResponse { - oneof response { - bool not_found = 1; - string failure = 2; - Table table = 3; - } -} - -message CreateTableRequest { - map configuration = 1; - string schema_name = 2; - Table table = 3; -} - -message CreateTableResponse { - oneof response { - bool success = 1; - string failure = 2; - } -} - -message AlterTableRequest { - map configuration = 1; - string schema_name = 2; - Table table = 3; -} - -message AlterTableResponse { - oneof response { - bool success = 1; - string failure = 2; - } -} - -message TruncateRequest { - map configuration = 1; - string schema_name = 2; - string table_name = 3; - string synced_column = 4; - google.protobuf.Timestamp utc_delete_before = 5; - optional SoftTruncate soft = 6; -} - -message SoftTruncate { - string deleted_column = 3; -} - -message TruncateResponse { - oneof response { - bool success = 1; - string failure = 2; - } -} - -message WriteBatchRequest { - map configuration = 1; - string schema_name = 2; - Table table = 3; - map keys = 4; - repeated string replace_files = 5; - repeated string update_files = 6; - repeated string delete_files = 7; - oneof file_params { - CsvFileParams csv = 8; - ParquetFileParams parquet = 9; - } -} - -message CsvFileParams { - Compression compression = 1; - Encryption encryption = 2; - string null_string = 3; - string unmodified_string = 4; -} - -message ParquetFileParams { - Encryption encryption = 1; -} - -enum Encryption { - NONE = 0; - AES = 1; -} - -enum BatchFileFormat { - CSV = 0; - PARQUET = 1; -} - -enum Compression { - OFF = 0; - ZSTD = 1; - GZIP = 2; -} - -message WriteBatchResponse { - oneof response { - bool success = 1; - string failure = 2; - } -} \ No newline at end of file diff --git a/v1/examples/destination_connector/java/src/main/java/destination/DestinationServiceImpl.java b/v1/examples/destination_connector/java/src/main/java/destination/DestinationServiceImpl.java deleted file mode 100644 index 54c40b6..0000000 --- a/v1/examples/destination_connector/java/src/main/java/destination/DestinationServiceImpl.java +++ /dev/null @@ -1,130 +0,0 @@ -package destination; - -import fivetran_sdk.*; -import io.grpc.stub.StreamObserver; - -import java.util.Arrays; -import java.util.Map; - -public class DestinationServiceImpl extends DestinationGrpc.DestinationImplBase { - - private final String INFO = "INFO"; - private final String WARNING = "WARNING"; - private final String SEVERE = "SEVERE"; - - @Override - public void configurationForm(ConfigurationFormRequest request, StreamObserver responseObserver) { - logMessage(INFO, "Fetching configuration form"); - responseObserver.onNext( - ConfigurationFormResponse.newBuilder() - .setSchemaSelectionSupported(true) - .setTableSelectionSupported(true) - .addAllFields(Arrays.asList( - FormField.newBuilder() - .setName("host").setLabel("Host").setRequired(true).setTextField(TextField.PlainText).build(), - FormField.newBuilder() - .setName("password").setLabel("Password").setRequired(true).setTextField(TextField.Password).build(), - FormField.newBuilder() - .setName("region").setLabel("AWS Region").setRequired(false).setDropdownField( - DropdownField.newBuilder().addAllDropdownField( - Arrays.asList("US-EAST", "US-WEST")).build() - ).build(), - FormField.newBuilder() - .setName("hidden").setLabel("my-hidden-value").setTextField(TextField.Hidden) - .build(), - FormField.newBuilder() - .setName("isPublic") - .setLabel("Public?") - .setDescription("Is this public?") - .setToggleField(ToggleField.newBuilder().build()) - .build() - )) - .addAllTests(Arrays.asList( - ConfigurationTest.newBuilder().setName("connect").setLabel("Tests connection").build(), - ConfigurationTest.newBuilder().setName("select").setLabel("Tests selection").build())) - .build()); - - responseObserver.onCompleted(); - } - - @Override - public void test(TestRequest request, StreamObserver responseObserver) { - Map configuration = request.getConfigurationMap(); - String testName = request.getName(); - String message = String.format("Test Name: %s", testName); - logMessage(INFO, message); - - responseObserver.onNext(TestResponse.newBuilder().setSuccess(true).build()); - responseObserver.onCompleted(); - } - - @Override - public void describeTable(DescribeTableRequest request, StreamObserver responseObserver) { - Map configuration = request.getConfigurationMap(); - - DescribeTableResponse response = DescribeTableResponse.newBuilder() - .setTable( - Table.newBuilder() - .setName(request.getTableName()) - .addAllColumns( - Arrays.asList( - Column.newBuilder().setName("a1").setType(DataType.UNSPECIFIED).setPrimaryKey(true).build(), - Column.newBuilder().setName("a2").setType(DataType.DOUBLE).build()) - ).build()).build(); - - responseObserver.onNext(response); - logMessage(SEVERE, "Sample Severe log: Completed describe Table method"); - responseObserver.onCompleted(); - } - - @Override - public void createTable(CreateTableRequest request, StreamObserver responseObserver) { - Map configuration = request.getConfigurationMap(); - - String message = "[CreateTable]: " - + request.getSchemaName() + " | " + request.getTable().getName() + " | " + request.getTable().getColumnsList(); - logMessage(INFO, message); - responseObserver.onNext(CreateTableResponse.newBuilder().setSuccess(true).build()); - responseObserver.onCompleted(); - } - - @Override - public void alterTable(AlterTableRequest request, StreamObserver responseObserver) { - Map configuration = request.getConfigurationMap(); - - String message = "[AlterTable]: " + - request.getSchemaName() + " | " + request.getTable().getName() + " | " + request.getTable().getColumnsList(); - logMessage(INFO, message); - responseObserver.onNext(AlterTableResponse.newBuilder().setSuccess(true).build()); - responseObserver.onCompleted(); - } - - @Override - public void truncate(TruncateRequest request, StreamObserver responseObserver) { - System.out.printf("[TruncateTable]: %s | %s | soft=%s%n", - request.getSchemaName(), request.getTableName(), request.hasSoft()); - responseObserver.onNext(TruncateResponse.newBuilder().setSuccess(true).build()); - responseObserver.onCompleted(); - } - - @Override - public void writeBatch(WriteBatchRequest request, StreamObserver responseObserver) { - String message = "[WriteBatch]: " + request.getSchemaName() + " | " + request.getTable().getName(); - logMessage(WARNING, String.format("Sample severe message: %s", message)); - for (String file : request.getReplaceFilesList()) { - System.out.println("Replace files: " + file); - } - for (String file : request.getUpdateFilesList()) { - System.out.println("Update files: " + file); - } - for (String file : request.getDeleteFilesList()) { - System.out.println("Delete files: " + file); - } - responseObserver.onNext(WriteBatchResponse.newBuilder().setSuccess(true).build()); - responseObserver.onCompleted(); - } - - private void logMessage(String level, String message){ - System.out.println(String.format("{\"level\":\"%s\", \"message\": \"%s\", \"message-origin\": \"sdk_destination\"}", level, message)); - } -} diff --git a/v1/examples/destination_connector/python/main.py b/v1/examples/destination_connector/python/main.py deleted file mode 100644 index 9de78ee..0000000 --- a/v1/examples/destination_connector/python/main.py +++ /dev/null @@ -1,95 +0,0 @@ -from concurrent import futures -import grpc -import read_csv -import sys -sys.path.append('sdk_pb2') - -from sdk_pb2 import destination_sdk_pb2 -from sdk_pb2 import common_pb2 -from sdk_pb2 import destination_sdk_pb2_grpc - - -INFO = "INFO" -WARNING = "WARNING" -SEVERE = "SEVERE" - -class DestinationImpl(destination_sdk_pb2_grpc.DestinationServicer): - def ConfigurationForm(self, request, context): - log_message(INFO, "Fetching Configuraiton form") - host = common_pb2.FormField(name="host", label="Host", required=True, - text_field=common_pb2.TextField.PlainText) - password = common_pb2.FormField(name="password", label="Password", required=True, - text_field=common_pb2.TextField.Password) - region = common_pb2.FormField(name="region", label="AWS Region", required=False, - dropdown_field=common_pb2.DropdownField(dropdown_field=["US-EAST", "US-WEST"])) - hidden = common_pb2.FormField(name="hidden", label="my-hidden-value", text_field=common_pb2.TextField.Hidden) - is_public = common_pb2.FormField(name="isPublic", label="Public?", description="Is this public?", - toggle_field=common_pb2.ToggleField()) - - connect_test = common_pb2.ConfigurationTest(name="connect", label="Tests connection") - select_test = common_pb2.ConfigurationTest(name="select", label="Tests selection") - return common_pb2.ConfigurationFormResponse( - schema_selection_supported=True, - table_selection_supported=True, - fields=[host, password, region, hidden, - is_public], - tests=[connect_test, select_test] - - ) - - def Test(self, request, context): - test_name = request.name - log_message(INFO, "test name: " + test_name) - return common_pb2.TestResponse(success=True) - - def CreateTable(self, request, context): - print("[CreateTable] :" + str(request.schema_name) + " | " + str(request.table.name) + " | " + str(request.table.columns)) - return destination_sdk_pb2.CreateTableResponse(success=True) - - def AlterTable(self, request, context): - res: destination_sdk_pb2.AlterTableResponse - - print("[AlterTable]: " + str(request.schema_name) + " | " + str(request.table.name) + " | " + str(request.table.columns)) - return destination_sdk_pb2.AlterTableResponse(success=True) - - def Truncate(self, request, context): - print("[TruncateTable]: " + str(request.schema_name) + " | " + str(request.schema_name) + " | soft" + str(request.soft)) - return destination_sdk_pb2.TruncateResponse(success=True) - - def WriteBatch(self, request, context): - for replace_file in request.replace_files: - print("replace files: " + str(replace_file)) - for update_file in request.update_files: - print("replace files: " + str(update_file)) - for delete_file in request.delete_files: - print("replace files: " + str(delete_file)) - - log_message(WARNING, "Data loading started for table " + request.table.name) - for key, value in request.keys.items(): - print("----------------------------------------------------------------------------") - print("Decrypting and printing file :" + str(key)) - print("----------------------------------------------------------------------------") - read_csv.decrypt_file(key, value) - log_message(INFO, "\nData loading completed for table " + request.table.name + "\n") - - res: destination_sdk_pb2.WriteBatchResponse = destination_sdk_pb2.WriteBatchResponse(success=True) - return res - - def DescribeTable(self, request, context): - column1 = common_pb2.Column(name="a1", type=common_pb2.DataType.UNSPECIFIED, primary_key=True) - column2 = common_pb2.Column(name="a2", type=common_pb2.DataType.DOUBLE) - table: common_pb2.Table = common_pb2.Table(name=request.table_name, columns=[column1, column2]) - log_message(SEVERE, "Sample severe message: Completed fetching table info") - return destination_sdk_pb2.DescribeTableResponse(not_found=False, table=table) - -def log_message(level, message): - print(f'{{"level":"{level}", "message": "{message}", "message-origin": "sdk_destination"}}') - -if __name__ == '__main__': - server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) - server.add_insecure_port('[::]:50052') - destination_sdk_pb2_grpc.add_DestinationServicer_to_server(DestinationImpl(), server) - server.start() - print("Destination gRPC server started...") - server.wait_for_termination() - print("Destination gRPC server terminated...") diff --git a/v1/examples/source_connector/golang/README.md b/v1/examples/source_connector/golang/README.md deleted file mode 100644 index f9a80f3..0000000 --- a/v1/examples/source_connector/golang/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# GoLang Connector gRPC Server -Run all commands from the golang folder root - -## Steps -``` -> cd v1/examples/source_connector/golang -> scripts/copy_protos.sh -> scripts/compile_protos.sh -> scripts/build.sh -> ./main -``` \ No newline at end of file diff --git a/v1/examples/source_connector/golang/golang_connector/main.go b/v1/examples/source_connector/golang/golang_connector/main.go deleted file mode 100644 index c15fe3d..0000000 --- a/v1/examples/source_connector/golang/golang_connector/main.go +++ /dev/null @@ -1,296 +0,0 @@ -package main - -import ( - context "context" - "encoding/json" - "flag" - "fmt" - "log" - "net" - "strconv" - - pb "fivetran.com/fivetran_sdk/proto" - "google.golang.org/grpc" - _ "google.golang.org/grpc/encoding/gzip" -) - -const INFO = "INFO" -const WARNING = "WARNING" -const SEVERE = "SEVERE" - -var port = flag.Int("port", 50051, "The server port") - -type MyState struct { - Cursor int32 `json:"cursor"` -} - -type server struct { - pb.UnimplementedConnectorServer -} - -func (s *server) Update(in *pb.UpdateRequest, stream pb.Connector_UpdateServer) error { - config := in.Configuration - selection := in.Selection - state_json := in.GetStateJson() - state := MyState{} - json.Unmarshal([]byte(state_json), &state) - - message := fmt.Sprintf("config: %s, selection: %s, state_json: %s, mystate: %+v", config, selection, state_json, state) - LogMessage(INFO, message) - - // -- Send a log message - stream.Send(&pb.UpdateResponse{ - Response: &pb.UpdateResponse_LogEntry{ - LogEntry: &pb.LogEntry{ - Level: pb.LogLevel_INFO, - Message: "Sync STARTING", - }, - }, - }) - - // -- Send UPSERT records - schemaName := "schema1" - for i := 0; i < 3; i++ { - stream.Send(&pb.UpdateResponse{ - Response: &pb.UpdateResponse_Operation{ - Operation: &pb.Operation{ - Op: &pb.Operation_Record{ - Record: &pb.Record{ - SchemaName: &schemaName, - TableName: "table1", - Type: pb.OpType_UPSERT, - Data: map[string]*pb.ValueType{ - "a1": {Inner: &pb.ValueType_String_{String_: "a-" + strconv.Itoa(i)}}, - "a2": {Inner: &pb.ValueType_Double{Double: float64(i) * float64(0.234)}}, - }, - }, - }, - }, - }, - }) - - state.Cursor++ - } - - LogMessage(INFO, "Completed sending upsert records") - - // -- Send UPDATE record - stream.Send(&pb.UpdateResponse{ - Response: &pb.UpdateResponse_Operation{ - Operation: &pb.Operation{ - Op: &pb.Operation_Record{ - Record: &pb.Record{ - SchemaName: &schemaName, - TableName: "table1", - Type: pb.OpType_UPDATE, - Data: map[string]*pb.ValueType{ - "a1": {Inner: &pb.ValueType_String_{String_: "a-0"}}, - "a2": {Inner: &pb.ValueType_Double{Double: float64(110.234)}}, - }, - }, - }, - }, - }, - }) - state.Cursor++ - - LogMessage(INFO, "Completed sending update records") - - // -- Send DELETE record - stream.Send(&pb.UpdateResponse{ - Response: &pb.UpdateResponse_Operation{ - Operation: &pb.Operation{ - Op: &pb.Operation_Record{ - Record: &pb.Record{ - SchemaName: &schemaName, - TableName: "table1", - Type: pb.OpType_DELETE, - Data: map[string]*pb.ValueType{ - "a1": {Inner: &pb.ValueType_String_{String_: "a-2"}}, - }, - }, - }, - }, - }, - }) - state.Cursor++ - - LogMessage(WARNING, "Sample warning message: Completed sending delete records") - - // Serialize state from struct to JSON string - new_state_json, _ := json.Marshal(state) - new_state := string(new_state_json) - log.Println("new_state: ", new_state) - - // -- Send Checkpoint - stream.Send(&pb.UpdateResponse{ - Response: &pb.UpdateResponse_Operation{ - Operation: &pb.Operation{ - Op: &pb.Operation_Checkpoint{ - Checkpoint: &pb.Checkpoint{ - StateJson: new_state, - }, - }, - }, - }, - }) - - // -- Send a log message - stream.Send(&pb.UpdateResponse{ - Response: &pb.UpdateResponse_LogEntry{ - LogEntry: &pb.LogEntry{ - Level: pb.LogLevel_INFO, - Message: "Sync DONE", - }, - }, - }) - - LogMessage(SEVERE, "Sample severe message: Update call completed") - // End the RPC call - return nil -} - -func (s *server) Schema(ctx context.Context, in *pb.SchemaRequest) (*pb.SchemaResponse, error) { - config := in.Configuration - log.Println(config) - - return &pb.SchemaResponse{ - Response: &pb.SchemaResponse_WithSchema{ - WithSchema: &pb.SchemaList{ - Schemas: []*pb.Schema{ - { - Name: "schema1", - Tables: []*pb.Table{ - { - Name: "table1", - Columns: []*pb.Column{ - { - Name: "a1", - Type: pb.DataType_UNSPECIFIED, - PrimaryKey: true, - }, - { - Name: "a2", - Type: pb.DataType_DOUBLE, - PrimaryKey: false, - }, - }, - }, - { - Name: "table2", - Columns: []*pb.Column{ - { - Name: "b1", - Type: pb.DataType_STRING, - PrimaryKey: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, nil -} - -func (s *server) ConfigurationForm(ctx context.Context, in *pb.ConfigurationFormRequest) (*pb.ConfigurationFormResponse, error) { - toggleDescription := "Is this public?" - - return &pb.ConfigurationFormResponse{ - SchemaSelectionSupported: true, - TableSelectionSupported: true, - Fields: []*pb.FormField{ - { - Name: "apikey", - Label: "API key", - Required: true, - Type: &pb.FormField_TextField{ - TextField: pb.TextField_PlainText, - }, - }, - { - Name: "password", - Label: "User password", - Required: true, - Type: &pb.FormField_TextField{ - TextField: pb.TextField_Password, - }, - }, - { - Name: "hidden", - Label: "my-hidden-value", - Type: &pb.FormField_TextField{ - TextField: pb.TextField_Hidden, - }, - }, - { - Name: "isPublic", - Label: "Public?", - Description: &toggleDescription, - Required: false, - Type: &pb.FormField_ToggleField{ - ToggleField: &pb.ToggleField{}, - }, - }, - { - Name: "region", - Label: "Region", - Required: true, - Type: &pb.FormField_DropdownField{ - DropdownField: &pb.DropdownField{ - DropdownField: []string{ - "US-EAST", "US-WEST", - }, - }, - }, - }, - }, - Tests: []*pb.ConfigurationTest{ - { - Name: "connect", - Label: "Test connection", - }, - { - Name: "select", - Label: "Test selection", - }, - }, - }, nil -} - -func (s *server) Test(ctx context.Context, in *pb.TestRequest) (*pb.TestResponse, error) { - config := in.Configuration - log.Println(config) - - log.Printf("test name: %v", in.Name) - return &pb.TestResponse{ - Response: &pb.TestResponse_Success{ - Success: true, - }, - }, nil -} - -func LogMessage(level string, message string) { - log := map[string]interface{}{ - "level": level, - "message": message, - "message-origin": "sdk_connector", - } - logJSON, _ := json.Marshal(log) - fmt.Println(string(logJSON)) -} - -func main() { - flag.Parse() - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - s := grpc.NewServer() - pb.RegisterConnectorServer(s, &server{}) - log.Printf("server listening at %v", lis.Addr()) - if err := s.Serve(lis); err != nil { - log.Fatalf("failed to serve: %v", err) - } -} diff --git a/v1/examples/source_connector/java/src/main/java/connector/ConnectorServiceImpl.java b/v1/examples/source_connector/java/src/main/java/connector/ConnectorServiceImpl.java deleted file mode 100644 index 9cbb5aa..0000000 --- a/v1/examples/source_connector/java/src/main/java/connector/ConnectorServiceImpl.java +++ /dev/null @@ -1,201 +0,0 @@ -package connector; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import fivetran_sdk.*; -import fivetran_sdk.Record; -import io.grpc.stub.StreamObserver; - -import java.util.*; - -public class ConnectorServiceImpl extends ConnectorGrpc.ConnectorImplBase { - private final String INFO = "INFO"; - private final String WARNING = "WARNING"; - private final String SEVERE = "SEVERE"; - @Override - public void configurationForm(ConfigurationFormRequest request, StreamObserver responseObserver) { - logMessage(INFO, "Started fetching configuration form"); - responseObserver.onNext( - ConfigurationFormResponse.newBuilder() - .setSchemaSelectionSupported(true) - .setTableSelectionSupported(true) - .addAllFields(Arrays.asList( - FormField.newBuilder() - .setName("apikey").setLabel("API key").setRequired(true).setTextField(TextField.PlainText).build(), - FormField.newBuilder() - .setName("password").setLabel("User Password").setRequired(true).setTextField(TextField.Password).build(), - FormField.newBuilder() - .setName("region").setLabel("AWS Region").setRequired(false).setDropdownField( - DropdownField.newBuilder().addAllDropdownField( - Arrays.asList("US-EAST", "US-WEST")).build() - ).build(), - FormField.newBuilder() - .setName("hidden").setLabel("my-hidden-value").setTextField(TextField.Hidden) - .build(), - FormField.newBuilder() - .setName("isPublic") - .setLabel("Public?") - .setDescription("Is this public?") - .setToggleField(ToggleField.newBuilder() - .build()) - .build() - )) - .addAllTests(Arrays.asList( - ConfigurationTest.newBuilder().setName("connect").setLabel("Tests connection").build(), - ConfigurationTest.newBuilder().setName("select").setLabel("Tests selection").build())) - .build()); - - logMessage(INFO, "Fetching configuration form completed"); - responseObserver.onCompleted(); - } - - @Override - public void test(TestRequest request, StreamObserver responseObserver) { - - Map configuration = request.getConfigurationMap(); - - // Name of the test to be run - String testName = request.getName(); - String message = String.format("test name: %s", testName); - logMessage(INFO, message); - - responseObserver.onNext(TestResponse.newBuilder().setSuccess(true).build()); - responseObserver.onCompleted(); - } - - @Override - public void schema(SchemaRequest request, StreamObserver responseObserver) { - - logMessage(WARNING, "Sample warning message while fetching schema"); - Map configuration = request.getConfigurationMap(); - - TableList tableList = TableList.newBuilder() - .addAllTables(Arrays.asList( - Table.newBuilder().setName("table1").addAllColumns( - Arrays.asList( - Column.newBuilder().setName("a1").setType(DataType.UNSPECIFIED).setPrimaryKey(true).build(), - Column.newBuilder().setName("a2").setType(DataType.DOUBLE).build()) - ).build(), - Table.newBuilder().setName("table2").addAllColumns( - Arrays.asList( - Column.newBuilder().setName("b1").setType(DataType.STRING).setPrimaryKey(true).build(), - Column.newBuilder().setName("b2").setType(DataType.UNSPECIFIED).build()) - ).build()) - ).build(); - - responseObserver.onNext(SchemaResponse.newBuilder().setWithoutSchema(tableList).build()); - responseObserver.onCompleted(); - } - - @Override - public void update(UpdateRequest request, StreamObserver responseObserver) { - Map configuration = request.getConfigurationMap(); - String state_json = request.hasStateJson() ? request.getStateJson() : "{}"; - Selection selection = request.hasSelection() ? request.getSelection() : null; - - ObjectMapper mapper = new ObjectMapper(); - UpdateResponse.Builder responseBuilder = UpdateResponse.newBuilder(); - - try { - State state = mapper.readValue(state_json, State.class); - - // -- Send a log message - responseBuilder.clear(); - responseObserver.onNext(responseBuilder - .setLogEntry(LogEntry.newBuilder() - .setLevel(LogLevel.INFO) - .setMessage("Sync STARTING") - .build()) - .build()); - - // -- Send UPSERT records - Operation.Builder operationBuilder = Operation.newBuilder(); - Record.Builder recordBuilder = Record.newBuilder(); - Map row = new HashMap<>(); - for (int i=0; i<3; i++) { - responseBuilder.clear(); - operationBuilder.clear(); - recordBuilder.clear(); - - row.clear(); - row.put("a1", ValueType.newBuilder().setString("a-" + i).build()); - row.put("a2", ValueType.newBuilder().setDouble(i * 0.234d).build()); - - responseObserver.onNext(responseBuilder - .setOperation(operationBuilder - .setRecord(recordBuilder - .setTableName("table1") - .setType(OpType.UPSERT) - .putAllData(row) - .build()) - .build()) - .build()); - - state.cursor += 1; - } - - // -- Send UPDATE record - responseBuilder.clear(); - operationBuilder.clear(); - recordBuilder.clear(); - row.clear(); - row.put("a1", ValueType.newBuilder().setString("a-0").build()); - row.put("a2", ValueType.newBuilder().setDouble(110.234d).build()); - responseObserver.onNext(responseBuilder - .setOperation(operationBuilder - .setRecord(recordBuilder - .setTableName("table1") - .setType(OpType.UPDATE) - .putAllData(row) - .build()) - .build()) - .build()); - state.cursor += 1; - - // -- Send DELETE record - responseBuilder.clear(); - operationBuilder.clear(); - recordBuilder.clear(); - row.clear(); - row.put("a1", ValueType.newBuilder().setString("a-2").build()); - responseObserver.onNext(responseBuilder - .setOperation(operationBuilder - .setRecord(recordBuilder - .setTableName("table1") - .setType(OpType.DELETE) - .putAllData(row) - .build()) - .build()) - .build()); - state.cursor += 1; - - // -- Send checkpoint - String newState = mapper.writeValueAsString(state); - Checkpoint checkpoint = Checkpoint.newBuilder().setStateJson(newState).build(); - operationBuilder.clear(); - responseObserver.onNext(responseBuilder - .setOperation(operationBuilder - .setCheckpoint(checkpoint).build()).build()); - - // -- Send a log message - responseBuilder.clear(); - responseObserver.onNext(responseBuilder - .setLogEntry(LogEntry.newBuilder() - .setLevel(LogLevel.INFO) - .setMessage("Sync DONE") - .build()) - .build()); - } catch (JsonProcessingException e) { - String message = e.getMessage(); - logMessage(SEVERE, message); - responseObserver.onError(e); - } - - // End the streaming RPC call - responseObserver.onCompleted(); - } - - private void logMessage(String level, String message) { - System.out.println(String.format("{\"level\":\"%s\", \"message\": \"%s\", \"message-origin\": \"sdk_connector\"}", level, message)); - } -} diff --git a/v1/examples/source_connector/nodejs/src/index.js b/v1/examples/source_connector/nodejs/src/index.js deleted file mode 100644 index 992946c..0000000 --- a/v1/examples/source_connector/nodejs/src/index.js +++ /dev/null @@ -1,169 +0,0 @@ -const grpc = require("@grpc/grpc-js"); -const PROTO_PATH_CONNECTOR = "./src/protos/connector_sdk.proto"; -var protoLoader = require("@grpc/proto-loader"); - -const options = { - keepCase: true, - longs: String, - enums: String, - defaults: true, - oneofs: true, -}; -var packageDefinitionConnector = protoLoader.loadSync(PROTO_PATH_CONNECTOR, options); - -const INFO = "INFO"; -const WARNING = "WARNING"; -const SEVERE = "SEVERE"; - -const protoDescriptor = grpc.loadPackageDefinition(packageDefinitionConnector); - -const connectorSdkProto = protoDescriptor.fivetran_sdk; - -const server = new grpc.Server(); - -const configurationForm = (call, callback) => { - logMessage(INFO, "Fetching configuration form") - callback(null, { - schema_selection_supported: true, - table_selection_supported: true, - fields: [ - { name: "apikey", label: "API key", required: true, text_field: "PlainText" }, - { name: "password", label: "User Password", required: true, text_field: "Password" }, - { name: "region", label: "AWS Region", required: false, dropdown_field: { dropdown_field : ["US-EAST","US-WEST"]} }, - { name: "hidden", label: "my-hidden-value", text_field:"Hidden" }, - { name: "isPublic", label: "Public?", description: "Is this public?", toggle_field: {} } - ], - tests: [ - { name: "connect", label: "Tests connection" }, - { name: "select", label: "Tests selection" } - ] - }); - }; - - // Implement the Test RPC method - const test = (call, callback) => { - const configuration = call.request.configuration; - const testName = call.request.name; - logMessage(INFO, `Test name: ${testName}`) - callback(null, { success: true }); - }; - - // Implement the Schema RPC method - const schema = (call, callback) => { - logMessage(INFO, "Fetching the schema from the implemented method") - const tableList = { - tables: [ - { - name: "table1", - columns: [ - { name: "a1", type: "UNSPECIFIED", primary_key: true }, - { name: "a2", type: "DOUBLE" } - ] - }, - { - name: "table2", - columns: [ - { name: "b1", type: "STRING", primary_key: true }, - { name: "b2", type: "UNSPECIFIED" } - ] - } - ] - }; - callback(null, { without_schema: tableList }); - }; - - - // Implement the update RPC method - const update = (call, callback) => { - const { configuration, state_json = '{}', selection } = call.request; - - const state = JSON.parse(state_json); - - const sendResponse = (response) => { - call.write(response); - }; - - const sendOperation = (operation) => { - sendResponse({ - operation: operation - }); - }; - - try { - // Send a log message - logMessage(WARNING, "Sample Warning message: Sync STARTING"); - - // Send UPSERT records - for (let i = 0; i < 3; i++) { - sendOperation({ - record: { - table_name: "table1", - type: "UPSERT", - data: { - "a1": {"string": `a-${i}`}, - "a2": {"double": i*0.234} - } - } - }); - state.cursor = (state.cursor || 0) + 1; - } - - // Send UPDATE record - sendOperation({ - record: { - table_name: "table1", - type: "UPDATE", - data: { - "a1": { "string": "a-0" }, - "a2": { "double": 110.234} - } - } - }); - state.cursor = (state.cursor || 0) + 1; - - // Send DELETE record - sendOperation({ - record: { - table_name: "table1", - type: "DELETE", - data: { - "a1": { "string" : "a-2" } - } - } - }); - state.cursor = (state.cursor || 0) + 1; - - // Send checkpoint - const newState = JSON.stringify(state); - sendOperation({ - checkpoint: { - state_json: newState - } - }); - - // Send a log message - logMessage(SEVERE, "Sample severe message: Sync done") - - } catch (error) { - callback(error); - } - - // End the streaming RPC call - call.end(); - }; - - function logMessage(level, message) { - console.log(`{"level":"${level}", "message": "${message}", "message-origin": "sdk_connector"}`); - } - - - server.addService(connectorSdkProto.Connector.service, {configurationForm, test, schema, update}) - - server.bindAsync( - '0.0.0.0'.concat(':').concat(50051), - grpc.ServerCredentials.createInsecure(), - (error, port) => { - console.log("Server running at http://127.0.0.1:50051"); - !error ? server.start() : console.log("Server failed with error: " + error) - } - ); \ No newline at end of file diff --git a/v1/examples/source_connector/python/main.py b/v1/examples/source_connector/python/main.py deleted file mode 100644 index 9367b52..0000000 --- a/v1/examples/source_connector/python/main.py +++ /dev/null @@ -1,168 +0,0 @@ -import grpc -from concurrent import futures -import json -import sys -sys.path.append('sdk_pb2') - -from sdk_pb2 import connector_sdk_pb2_grpc -from sdk_pb2 import common_pb2 -from sdk_pb2 import connector_sdk_pb2 - -INFO = "INFO" -WARNING = "WARNING" -SEVERE = "SEVERE" - -class ConnectorService(connector_sdk_pb2_grpc.ConnectorServicer): - def ConfigurationForm(self, request, context): - log_message(INFO, "Fetching configuration form") - form_fields = common_pb2.ConfigurationFormResponse(schema_selection_supported=True, - table_selection_supported=True) - form_fields.fields.add(name="apiKey", label="API Key", required=True, text_field=common_pb2.TextField.PlainText) - form_fields.fields.add(name="password", label="User Password", required=True, text_field=common_pb2.TextField.Password) - - form_fields.fields.add( - name="region", - label="AWS Region", - required=False, - dropdown_field=common_pb2.DropdownField(dropdown_field=["US-EAST", "US-WEST"]) - ) - - form_fields.fields.add(name="hidden", label="my-hidden-value", text_field=common_pb2.TextField.Hidden) - form_fields.fields.add(name="isPublic", label="Public?", description="Is this public?", toggle_field=common_pb2.ToggleField()) - - # add setup tests - form_fields.tests.add(name="connection_test", label="Tests connection") - - return form_fields - - def Test(self, request, context): - configuration = request.configuration - # Name of the test to be run - test_name = request.name - - log_message(INFO, "Test Name: " + str(test_name)) - - return common_pb2.TestResponse(success=True) - - def Schema(self, request, context): - table_list = common_pb2.TableList() - t1 = table_list.tables.add(name="table1") - t1.columns.add(name="a1", type=common_pb2.DataType.UNSPECIFIED, primary_key=True) - t1.columns.add(name="a2", type=common_pb2.DataType.DOUBLE) - - t2 = table_list.tables.add(name="table2") - t2.columns.add(name="b1", type=common_pb2.DataType.UNSPECIFIED, primary_key=True) - t2.columns.add(name="b2", type=common_pb2.DataType.UNSPECIFIED) - - return connector_sdk_pb2.SchemaResponse(without_schema=table_list) - - def Update(self, request, context): - - state_json = "{}" - if request.HasField('state_json'): - state_json = request.state_json - - state = json.loads(state_json) - if state.get("cursor") is None: - state["cursor"] = 0 - - # -- Send UPSERT records - for t in range(0, 3): - operation = connector_sdk_pb2.Operation() - val1 = common_pb2.ValueType() - val1.string = "a-" + str(t) - - val2 = common_pb2.ValueType() - val2.double = t * 0.234 - - record = connector_sdk_pb2.Record() - record.type = common_pb2.OpType.UPSERT - record.table_name = "table1" - record.data["a1"].CopyFrom(val1) - record.data["a2"].CopyFrom(val2) - state["cursor"] += 1 - - operation.record.CopyFrom(record) - yield connector_sdk_pb2.UpdateResponse(operation=operation) - - # -- Send UPSERT record for table2 - operation = connector_sdk_pb2.Operation() - val1 = common_pb2.ValueType() - val1.string = "b1" - val2 = common_pb2.ValueType() - val2.string = "ben" - record = connector_sdk_pb2.Record() - record.type = common_pb2.OpType.UPSERT - record.table_name = "table2" - record.data["b1"].CopyFrom(val1) - record.data["b2"].CopyFrom(val2) - state["cursor"] += 1 - - operation.record.CopyFrom(record) - yield connector_sdk_pb2.UpdateResponse(operation=operation) - - # -- Send UPDATE record - operation = connector_sdk_pb2.Operation() - val1 = common_pb2.ValueType() - val1.string = "a-0" - - val2 = common_pb2.ValueType() - val2.double = 110.234 - - record = connector_sdk_pb2.Record() - record.type = common_pb2.OpType.UPDATE - record.table_name = "table1" - record.data["a1"].CopyFrom(val1) - record.data["a2"].CopyFrom(val2) - - operation.record.CopyFrom(record) - yield connector_sdk_pb2.UpdateResponse(operation=operation) - state["cursor"] += 1 - - log_message(WARNING, "Completed sending update records") - - # -- Send DELETE record - operation = connector_sdk_pb2.Operation() - val1 = common_pb2.ValueType() - val1.string = "a-2" - - record = connector_sdk_pb2.Record() - record.type = common_pb2.OpType.DELETE - record.table_name = "table1" - record.data["a1"].CopyFrom(val1) - - operation.record.CopyFrom(record) - yield connector_sdk_pb2.UpdateResponse(operation=operation) - state["cursor"] += 1 - - checkpoint = connector_sdk_pb2.Checkpoint() - checkpoint.state_json = json.dumps(state) - checkpoint_operation = connector_sdk_pb2.Operation() - checkpoint_operation.checkpoint.CopyFrom(checkpoint) - yield connector_sdk_pb2.UpdateResponse(operation=checkpoint_operation) - - log = connector_sdk_pb2.LogEntry() - log.level = connector_sdk_pb2.LogLevel.INFO - log.message = "Sync Done" - yield connector_sdk_pb2.UpdateResponse(log_entry=log) - - log_message(SEVERE, "Sending severe log: Completed Update method") - - -def log_message(level, message): - print(f'{{"level":"{level}", "message": "{message}", "message-origin": "sdk_connector"}}') - - -def start_server(): - server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) - connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(ConnectorService(), server) - server.add_insecure_port('[::]:50051') - server.start() - print("Server started...") - server.wait_for_termination() - print("Server terminated.") - - -if __name__ == '__main__': - print("Starting the server...") - start_server() \ No newline at end of file diff --git a/v2/examples/destination_connector/java/.gitattributes b/v2/examples/destination_connector/java/.gitattributes deleted file mode 100644 index 00a51af..0000000 --- a/v2/examples/destination_connector/java/.gitattributes +++ /dev/null @@ -1,6 +0,0 @@ -# -# https://help.github.com/articles/dealing-with-line-endings/ -# -# These are explicitly windows files and should use crlf -*.bat text eol=crlf - diff --git a/v2/examples/destination_connector/java/Dockerfile b/v2/examples/destination_connector/java/Dockerfile deleted file mode 100644 index d6b64e2..0000000 --- a/v2/examples/destination_connector/java/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM openjdk:11-jre-slim - -EXPOSE 50052 - -RUN mkdir /app - -COPY build/libs/*.jar /app/JavaDestination.jar - -ENTRYPOINT ["java", "-jar", "--illegal-access=debug", "--add-opens=java.base/java.io=ALL-UNNAMED", "--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.nio.charset=ALL-UNNAMED", "--add-opens=java.base/java.util=ALL-UNNAMED", "/app/JavaDestination.jar"] diff --git a/v2/examples/destination_connector/java/README.md b/v2/examples/destination_connector/java/README.md deleted file mode 100644 index cf8e93e..0000000 --- a/v2/examples/destination_connector/java/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Java Destination Example - -## Pre-requisites -- JDK v17 -- Gradle 8 - -## Steps -1. Copy proto files from the root folder -``` -> gradle copyProtos -``` -2. Build the Jar -``` -> gradle jar -``` -3. Run the Jar -``` -> java -jar build/libs/JavaDestination.jar -``` diff --git a/v2/examples/destination_connector/java/build.gradle b/v2/examples/destination_connector/java/build.gradle deleted file mode 100644 index a25f69f..0000000 --- a/v2/examples/destination_connector/java/build.gradle +++ /dev/null @@ -1,85 +0,0 @@ -plugins { - // Provide convenience executables for trying out the examples. - id 'application' - // ASSUMES GRADLE 5.6 OR HIGHER. Use plugin version 0.8.10 with earlier gradle versions - id 'com.google.protobuf' version '0.9.1' - // Generate IntelliJ IDEA's .idea & .iml project files - id 'idea' - id 'java' -} - -repositories { - maven { // The google mirror is less flaky than mavenCentral() - url "https://maven-central.storage-download.googleapis.com/maven2/" - - artifactUrls "https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/" - } - mavenCentral() -} - -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -def grpcVersion = '1.61.1' -def protobufVersion = '3.25.2' -def protocVersion = protobufVersion - -dependencies { - implementation "io.grpc:grpc-protobuf:${grpcVersion}" - implementation "io.grpc:grpc-stub:${grpcVersion}" - compileOnly "org.apache.tomcat:annotations-api:6.0.53" - - implementation "com.google.protobuf:protobuf-java-util:${protobufVersion}" - - runtimeOnly "io.grpc:grpc-netty-shaded:${grpcVersion}" - - implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2' - implementation "com.fasterxml.jackson.core:jackson-core:2.15.2" - - implementation 'com.github.luben:zstd-jni:1.5.5-11' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.2.3' -} - -protobuf { - protoc { artifact = "com.google.protobuf:protoc:${protocVersion}" } - plugins { - grpc { artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}" } - } - generateProtoTasks { - all()*.plugins { grpc {} } - } -} - -// Inform IDEs like IntelliJ IDEA, Eclipse or NetBeans about the generated code. -sourceSets { - main { - java { - srcDirs 'build/generated/source/proto/main/grpc' - srcDirs 'build/generated/source/proto/main/java' - } - } -} - -application { - mainClass = 'destination.JavaDestination' -} - -tasks.register('copyProtos', Copy) { - from file("$rootDir/../../..") - into file("src/main/proto/") - include "*.proto" -} - -jar { - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - - manifest { - attributes( - 'Main-Class' : 'destination.JavaDestination' - ) - } - - from { - configurations.runtimeClasspath.filter{ it.exists() }.collect { it.isDirectory() ? it : zipTree(it) } - } -} diff --git a/v2/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.jar b/v2/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 41d9927..0000000 Binary files a/v2/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/v2/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.properties b/v2/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 00e33ed..0000000 --- a/v2/examples/destination_connector/java/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/v2/examples/destination_connector/java/gradlew b/v2/examples/destination_connector/java/gradlew deleted file mode 100755 index 1b6c787..0000000 --- a/v2/examples/destination_connector/java/gradlew +++ /dev/null @@ -1,234 +0,0 @@ -#!/bin/sh - -# -# Copyright © 2015-2021 the original authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -# -# Gradle start up script for POSIX generated by Gradle. -# -# Important for running: -# -# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is -# noncompliant, but you have some other compliant shell such as ksh or -# bash, then to run this script, type that shell name before the whole -# command line, like: -# -# ksh Gradle -# -# Busybox and similar reduced shells will NOT work, because this script -# requires all of these POSIX shell features: -# * functions; -# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», -# «${var#prefix}», «${var%suffix}», and «$( cmd )»; -# * compound commands having a testable exit status, especially «case»; -# * various built-in commands including «command», «set», and «ulimit». -# -# Important for patching: -# -# (2) This script targets any POSIX shell, so it avoids extensions provided -# by Bash, Ksh, etc; in particular arrays are avoided. -# -# The "traditional" practice of packing multiple parameters into a -# space-separated string is a well documented source of bugs and security -# problems, so this is (mostly) avoided, by progressively accumulating -# options in "$@", and eventually passing that to Java. -# -# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, -# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; -# see the in-line comments for details. -# -# There are tweaks for specific operating systems such as AIX, CygWin, -# Darwin, MinGW, and NonStop. -# -# (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt -# within the Gradle project. -# -# You can find Gradle at https://github.com/gradle/gradle/. -# -############################################################################## - -# Attempt to set APP_HOME - -# Resolve links: $0 may be a link -app_path=$0 - -# Need this for daisy-chained symlinks. -while - APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path - [ -h "$app_path" ] -do - ls=$( ls -ld "$app_path" ) - link=${ls#*' -> '} - case $link in #( - /*) app_path=$link ;; #( - *) app_path=$APP_HOME$link ;; - esac -done - -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" -APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD=maximum - -warn () { - echo "$*" -} >&2 - -die () { - echo - echo "$*" - echo - exit 1 -} >&2 - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "$( uname )" in #( - CYGWIN* ) cygwin=true ;; #( - Darwin* ) darwin=true ;; #( - MSYS* | MINGW* ) msys=true ;; #( - NONSTOP* ) nonstop=true ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD=$JAVA_HOME/jre/sh/java - else - JAVACMD=$JAVA_HOME/bin/java - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then - case $MAX_FD in #( - max*) - MAX_FD=$( ulimit -H -n ) || - warn "Could not query maximum file descriptor limit" - esac - case $MAX_FD in #( - '' | soft) :;; #( - *) - ulimit -n "$MAX_FD" || - warn "Could not set maximum file descriptor limit to $MAX_FD" - esac -fi - -# Collect all arguments for the java command, stacking in reverse order: -# * args from the command line -# * the main class name -# * -classpath -# * -D...appname settings -# * --module-path (only if needed) -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. - -# For Cygwin or MSYS, switch paths to Windows format before running java -if "$cygwin" || "$msys" ; then - APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) - - JAVACMD=$( cygpath --unix "$JAVACMD" ) - - # Now convert the arguments - kludge to limit ourselves to /bin/sh - for arg do - if - case $arg in #( - -*) false ;; # don't mess with options #( - /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath - [ -e "$t" ] ;; #( - *) false ;; - esac - then - arg=$( cygpath --path --ignore --mixed "$arg" ) - fi - # Roll the args list around exactly as many times as the number of - # args, so each arg winds up back in the position where it started, but - # possibly modified. - # - # NB: a `for` loop captures its iteration list before it begins, so - # changing the positional parameters here affects neither the number of - # iterations, nor the values presented in `arg`. - shift # remove old arg - set -- "$@" "$arg" # push replacement arg - done -fi - -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. - -set -- \ - "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ - "$@" - -# Use "xargs" to parse quoted args. -# -# With -n1 it outputs one arg per line, with the quotes and backslashes removed. -# -# In Bash we could simply go: -# -# readarray ARGS < <( xargs -n1 <<<"$var" ) && -# set -- "${ARGS[@]}" "$@" -# -# but POSIX shell has neither arrays nor command substitution, so instead we -# post-process each arg (as a line of input to sed) to backslash-escape any -# character that might be a shell metacharacter, then use eval to reverse -# that process (while maintaining the separation between arguments), and wrap -# the whole thing up as a single "set" statement. -# -# This will of course break if any of these variables contains a newline or -# an unmatched quote. -# - -eval "set -- $( - printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | - xargs -n1 | - sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | - tr '\n' ' ' - )" '"$@"' - -exec "$JAVACMD" "$@" diff --git a/v2/examples/destination_connector/java/gradlew.bat b/v2/examples/destination_connector/java/gradlew.bat deleted file mode 100644 index 107acd3..0000000 --- a/v2/examples/destination_connector/java/gradlew.bat +++ /dev/null @@ -1,89 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/v2/examples/destination_connector/java/settings.gradle b/v2/examples/destination_connector/java/settings.gradle deleted file mode 100644 index e1c0175..0000000 --- a/v2/examples/destination_connector/java/settings.gradle +++ /dev/null @@ -1,10 +0,0 @@ -/* - * This file was generated by the Gradle 'init' task. - * - * The settings file is used to specify which projects to include in your build. - * - * Detailed information about configuring a multi-project build in Gradle can be found - * in the user manual at https://docs.gradle.org/7.4.1/userguide/multi_project_builds.html - */ - -rootProject.name = 'JavaDestination' diff --git a/v2/examples/destination_connector/java/src/main/java/destination/JavaDestination.java b/v2/examples/destination_connector/java/src/main/java/destination/JavaDestination.java deleted file mode 100644 index 95ad99c..0000000 --- a/v2/examples/destination_connector/java/src/main/java/destination/JavaDestination.java +++ /dev/null @@ -1,22 +0,0 @@ -package destination; - -import io.grpc.*; - -import java.io.IOException; - -/** - * Example Plugin Connector (gRPC server) - * In production, it will be stored as a container image - */ -public class JavaDestination { - - public static void main(String[] args) throws InterruptedException, IOException { - Server server = ServerBuilder - .forPort(50052) - .addService(new DestinationServiceImpl()).build(); - - server.start(); - System.out.println("Destination gRPC server started"); - server.awaitTermination(); - } -} diff --git a/v2/examples/destination_connector/python/README.md b/v2/examples/destination_connector/python/README.md deleted file mode 100644 index f154a71..0000000 --- a/v2/examples/destination_connector/python/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Python Destination Example - -## Pre-requisites -- Python 3.9 or later - -## Steps -- Run the build.sh file to copy protos, install python dependencies in virtual environment -```commandline -sh build.sh -``` - -- Execute `run.sh` to run the connector -```commandline -sh run.sh -``` \ No newline at end of file diff --git a/v2/examples/destination_connector/python/build.sh b/v2/examples/destination_connector/python/build.sh deleted file mode 100755 index 77bf52c..0000000 --- a/v2/examples/destination_connector/python/build.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -#Create virtual environment -python3 -m venv destination_run - -#Activate virtual environment -source destination_run/bin/activate - -# Make a directory protos -mkdir -p protos - -# Copy proto files t oprotos directory -cp ../../../*.proto protos/ - -# Install the required packages -pip install -r requirements.txt - -# Make a directory sdk_pb2 -mkdir -p sdk_pb2 - -# Generate grpc python code and store it in sdk_pb2 -python -m grpc_tools.protoc \ - --proto_path=./protos/ \ - --python_out=sdk_pb2 \ - --pyi_out=sdk_pb2 \ - --grpc_python_out=sdk_pb2 protos/*.proto - -# Deactivate virtual environment -deactivate \ No newline at end of file diff --git a/v2/examples/destination_connector/python/read_csv.py b/v2/examples/destination_connector/python/read_csv.py deleted file mode 100644 index 56a9216..0000000 --- a/v2/examples/destination_connector/python/read_csv.py +++ /dev/null @@ -1,32 +0,0 @@ -from zstandard import ZstdDecompressor -from Crypto.Cipher import AES -import csv - - -# AES decryption function -def aes_decrypt(key, ciphertext): - cipher = AES.new(key, AES.MODE_CBC, iv=ciphertext[:AES.block_size]) - plaintext = cipher.decrypt(ciphertext[AES.block_size:]) - return plaintext.rstrip(b'\0') - - -# Zstandard decompression function -def zstd_decompress(compressed_data): - decompressor = ZstdDecompressor() - decompressed_data = decompressor.decompressobj().decompress(compressed_data) - return decompressed_data - - -# Read the encrypted and compressed data -def decrypt_file(input_file_path, value): - with open(input_file_path, 'rb') as file: - encrypted_and_compressed_data = file.read() - decrypted_data = aes_decrypt(value, encrypted_and_compressed_data) - decompressed_data = zstd_decompress(decrypted_data) - csv_data = decompressed_data.decode('utf-8') - csv_reader = csv.reader(csv_data.splitlines()) - headers = next(csv_reader) - print(f"{' | '.join(headers)}") - print('-' * (len(headers) * 15)) - for row in csv_reader: - print(f"{' | '.join(row)}") diff --git a/v2/examples/destination_connector/python/requirements.txt b/v2/examples/destination_connector/python/requirements.txt deleted file mode 100644 index 9e5b8e3..0000000 --- a/v2/examples/destination_connector/python/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -grpcio==1.60.1 -grpcio-tools==1.60.1 -protobuf==4.25.3 -google~=3.0.0 -pip~=23.0.1 -setuptools~=65.5.0 -zstandard~=0.22.0 -pycryptodome==3.19.1 \ No newline at end of file diff --git a/v2/examples/destination_connector/python/run.sh b/v2/examples/destination_connector/python/run.sh deleted file mode 100755 index d90200f..0000000 --- a/v2/examples/destination_connector/python/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -source destination_run/bin/activate -python main.py -deactivate \ No newline at end of file diff --git a/v2/examples/source_connector/golang/Dockerfile b/v2/examples/source_connector/golang/Dockerfile deleted file mode 100644 index ceb9c91..0000000 --- a/v2/examples/source_connector/golang/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM golang:1.18-alpine - -WORKDIR /app - -# We want to populate the module cache based on the go.{mod,sum} files. -COPY go.mod . -COPY go.sum . - -RUN go mod download - -COPY . . - -# Build the Go app -RUN go build -o ./out/golang_connector ./golang_connector - -# This container exposes port to the outside world -EXPOSE 50051 - -# Run the binary program produced by `go install` -CMD ["./out/golang_connector"] \ No newline at end of file diff --git a/v2/examples/source_connector/golang/go.mod b/v2/examples/source_connector/golang/go.mod deleted file mode 100644 index 61bd712..0000000 --- a/v2/examples/source_connector/golang/go.mod +++ /dev/null @@ -1,15 +0,0 @@ -module fivetran.com/fivetran_sdk - -go 1.21 - -require ( - google.golang.org/grpc v1.65.0 - google.golang.org/protobuf v1.35.1 -) - -require ( - golang.org/x/net v0.30.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect -) diff --git a/v2/examples/source_connector/golang/go.sum b/v2/examples/source_connector/golang/go.sum deleted file mode 100644 index 4d4de4e..0000000 --- a/v2/examples/source_connector/golang/go.sum +++ /dev/null @@ -1,14 +0,0 @@ -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= diff --git a/v2/examples/source_connector/golang/scripts/build.sh b/v2/examples/source_connector/golang/scripts/build.sh deleted file mode 100755 index 947a8b2..0000000 --- a/v2/examples/source_connector/golang/scripts/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -mkdir proto -./scripts/copy_protos.sh -./scripts/compile_protos.sh -go build golang_connector/main.go \ No newline at end of file diff --git a/v2/examples/source_connector/golang/scripts/compile_protos.sh b/v2/examples/source_connector/golang/scripts/compile_protos.sh deleted file mode 100755 index 925c3bc..0000000 --- a/v2/examples/source_connector/golang/scripts/compile_protos.sh +++ /dev/null @@ -1,8 +0,0 @@ -PATH="${PATH}:${HOME}/go/bin" protoc \ - --proto_path=proto \ - --go_out=proto \ - --go_opt=paths=source_relative \ - --go-grpc_out=proto \ - --go-grpc_opt=paths=source_relative \ - common_v2.proto \ - source_connector_sdk_v2.proto diff --git a/v2/examples/source_connector/golang/scripts/copy_protos.sh b/v2/examples/source_connector/golang/scripts/copy_protos.sh deleted file mode 100755 index a6f1a14..0000000 --- a/v2/examples/source_connector/golang/scripts/copy_protos.sh +++ /dev/null @@ -1 +0,0 @@ -cp ../../../*.proto proto/ diff --git a/v2/examples/source_connector/golang/scripts/run.sh b/v2/examples/source_connector/golang/scripts/run.sh deleted file mode 100755 index 87c9079..0000000 --- a/v2/examples/source_connector/golang/scripts/run.sh +++ /dev/null @@ -1 +0,0 @@ -go run golang_connector/main.go \ No newline at end of file diff --git a/v2/examples/source_connector/java/.gitattributes b/v2/examples/source_connector/java/.gitattributes deleted file mode 100644 index 00a51af..0000000 --- a/v2/examples/source_connector/java/.gitattributes +++ /dev/null @@ -1,6 +0,0 @@ -# -# https://help.github.com/articles/dealing-with-line-endings/ -# -# These are explicitly windows files and should use crlf -*.bat text eol=crlf - diff --git a/v2/examples/source_connector/java/Dockerfile b/v2/examples/source_connector/java/Dockerfile deleted file mode 100644 index 85e2d08..0000000 --- a/v2/examples/source_connector/java/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM openjdk:11-jre-slim - -EXPOSE 50051 - -RUN mkdir /app - -COPY build/libs/*.jar /app/JavaConnector.jar - -ENTRYPOINT ["java", "-jar", "--illegal-access=debug", "--add-opens=java.base/java.io=ALL-UNNAMED", "--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.nio.charset=ALL-UNNAMED", "--add-opens=java.base/java.util=ALL-UNNAMED", "/app/JavaConnector.jar"] diff --git a/v2/examples/source_connector/java/README.md b/v2/examples/source_connector/java/README.md deleted file mode 100644 index c81d57a..0000000 --- a/v2/examples/source_connector/java/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Java Connector Example - -## Pre-requisites -- JDK v17 -- Gradle 8 - -## Steps -1. Copy proto files from the root folder -``` -> gradle copyProtos -``` -2. Build the Jar -``` -> gradle jar -``` -3. Run the Jar -``` -> java -jar build/libs/JavaConnector.jar -``` diff --git a/v2/examples/source_connector/java/build.gradle b/v2/examples/source_connector/java/build.gradle deleted file mode 100644 index a6b0153..0000000 --- a/v2/examples/source_connector/java/build.gradle +++ /dev/null @@ -1,82 +0,0 @@ -plugins { - // Provide convenience executables for trying out the examples. - id 'application' - // ASSUMES GRADLE 5.6 OR HIGHER. Use plugin version 0.8.10 with earlier gradle versions - id 'com.google.protobuf' version '0.9.1' - // Generate IntelliJ IDEA's .idea & .iml project files - id 'idea' - id 'java' -} - -repositories { - maven { // The google mirror is less flaky than mavenCentral() - url "https://maven-central.storage-download.googleapis.com/maven2/" - - artifactUrls "https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/" - } - mavenCentral() -} - -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -def grpcVersion = '1.59.1' -def protobufVersion = '3.25.1' -def protocVersion = protobufVersion - -dependencies { - implementation "io.grpc:grpc-protobuf:${grpcVersion}" - implementation "io.grpc:grpc-stub:${grpcVersion}" - compileOnly "org.apache.tomcat:annotations-api:6.0.53" - - implementation "com.google.protobuf:protobuf-java-util:${protobufVersion}" - - runtimeOnly "io.grpc:grpc-netty-shaded:${grpcVersion}" - - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1' - implementation "com.fasterxml.jackson.core:jackson-core:2.14.1" -} - -protobuf { - protoc { artifact = "com.google.protobuf:protoc:${protocVersion}" } - plugins { - grpc { artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}" } - } - generateProtoTasks { - all()*.plugins { grpc {} } - } -} - -// Inform IDEs like IntelliJ IDEA, Eclipse or NetBeans about the generated code. -sourceSets { - main { - java { - srcDirs 'build/generated/source/proto/main/grpc' - srcDirs 'build/generated/source/proto/main/java' - } - } -} - -application { - mainClass = 'connector.JavaConnector' -} - -tasks.register('copyProtos', Copy) { - from file("$rootDir/../../..") - into file("src/main/proto/") - include "*.proto" -} - -jar { - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - - manifest { - attributes( - 'Main-Class' : 'connector.JavaConnector' - ) - } - - from { - configurations.runtimeClasspath.filter{ it.exists() }.collect { it.isDirectory() ? it : zipTree(it) } - } -} \ No newline at end of file diff --git a/v2/examples/source_connector/java/gradle/wrapper/gradle-wrapper.jar b/v2/examples/source_connector/java/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 41d9927..0000000 Binary files a/v2/examples/source_connector/java/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/v2/examples/source_connector/java/gradle/wrapper/gradle-wrapper.properties b/v2/examples/source_connector/java/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 00e33ed..0000000 --- a/v2/examples/source_connector/java/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/v2/examples/source_connector/java/gradlew b/v2/examples/source_connector/java/gradlew deleted file mode 100755 index 1b6c787..0000000 --- a/v2/examples/source_connector/java/gradlew +++ /dev/null @@ -1,234 +0,0 @@ -#!/bin/sh - -# -# Copyright © 2015-2021 the original authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -# -# Gradle start up script for POSIX generated by Gradle. -# -# Important for running: -# -# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is -# noncompliant, but you have some other compliant shell such as ksh or -# bash, then to run this script, type that shell name before the whole -# command line, like: -# -# ksh Gradle -# -# Busybox and similar reduced shells will NOT work, because this script -# requires all of these POSIX shell features: -# * functions; -# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», -# «${var#prefix}», «${var%suffix}», and «$( cmd )»; -# * compound commands having a testable exit status, especially «case»; -# * various built-in commands including «command», «set», and «ulimit». -# -# Important for patching: -# -# (2) This script targets any POSIX shell, so it avoids extensions provided -# by Bash, Ksh, etc; in particular arrays are avoided. -# -# The "traditional" practice of packing multiple parameters into a -# space-separated string is a well documented source of bugs and security -# problems, so this is (mostly) avoided, by progressively accumulating -# options in "$@", and eventually passing that to Java. -# -# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, -# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; -# see the in-line comments for details. -# -# There are tweaks for specific operating systems such as AIX, CygWin, -# Darwin, MinGW, and NonStop. -# -# (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt -# within the Gradle project. -# -# You can find Gradle at https://github.com/gradle/gradle/. -# -############################################################################## - -# Attempt to set APP_HOME - -# Resolve links: $0 may be a link -app_path=$0 - -# Need this for daisy-chained symlinks. -while - APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path - [ -h "$app_path" ] -do - ls=$( ls -ld "$app_path" ) - link=${ls#*' -> '} - case $link in #( - /*) app_path=$link ;; #( - *) app_path=$APP_HOME$link ;; - esac -done - -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" -APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD=maximum - -warn () { - echo "$*" -} >&2 - -die () { - echo - echo "$*" - echo - exit 1 -} >&2 - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "$( uname )" in #( - CYGWIN* ) cygwin=true ;; #( - Darwin* ) darwin=true ;; #( - MSYS* | MINGW* ) msys=true ;; #( - NONSTOP* ) nonstop=true ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD=$JAVA_HOME/jre/sh/java - else - JAVACMD=$JAVA_HOME/bin/java - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then - case $MAX_FD in #( - max*) - MAX_FD=$( ulimit -H -n ) || - warn "Could not query maximum file descriptor limit" - esac - case $MAX_FD in #( - '' | soft) :;; #( - *) - ulimit -n "$MAX_FD" || - warn "Could not set maximum file descriptor limit to $MAX_FD" - esac -fi - -# Collect all arguments for the java command, stacking in reverse order: -# * args from the command line -# * the main class name -# * -classpath -# * -D...appname settings -# * --module-path (only if needed) -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. - -# For Cygwin or MSYS, switch paths to Windows format before running java -if "$cygwin" || "$msys" ; then - APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) - - JAVACMD=$( cygpath --unix "$JAVACMD" ) - - # Now convert the arguments - kludge to limit ourselves to /bin/sh - for arg do - if - case $arg in #( - -*) false ;; # don't mess with options #( - /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath - [ -e "$t" ] ;; #( - *) false ;; - esac - then - arg=$( cygpath --path --ignore --mixed "$arg" ) - fi - # Roll the args list around exactly as many times as the number of - # args, so each arg winds up back in the position where it started, but - # possibly modified. - # - # NB: a `for` loop captures its iteration list before it begins, so - # changing the positional parameters here affects neither the number of - # iterations, nor the values presented in `arg`. - shift # remove old arg - set -- "$@" "$arg" # push replacement arg - done -fi - -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. - -set -- \ - "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ - "$@" - -# Use "xargs" to parse quoted args. -# -# With -n1 it outputs one arg per line, with the quotes and backslashes removed. -# -# In Bash we could simply go: -# -# readarray ARGS < <( xargs -n1 <<<"$var" ) && -# set -- "${ARGS[@]}" "$@" -# -# but POSIX shell has neither arrays nor command substitution, so instead we -# post-process each arg (as a line of input to sed) to backslash-escape any -# character that might be a shell metacharacter, then use eval to reverse -# that process (while maintaining the separation between arguments), and wrap -# the whole thing up as a single "set" statement. -# -# This will of course break if any of these variables contains a newline or -# an unmatched quote. -# - -eval "set -- $( - printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | - xargs -n1 | - sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | - tr '\n' ' ' - )" '"$@"' - -exec "$JAVACMD" "$@" diff --git a/v2/examples/source_connector/java/gradlew.bat b/v2/examples/source_connector/java/gradlew.bat deleted file mode 100644 index 107acd3..0000000 --- a/v2/examples/source_connector/java/gradlew.bat +++ /dev/null @@ -1,89 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/v2/examples/source_connector/java/settings.gradle b/v2/examples/source_connector/java/settings.gradle deleted file mode 100644 index ac52d97..0000000 --- a/v2/examples/source_connector/java/settings.gradle +++ /dev/null @@ -1,10 +0,0 @@ -/* - * This file was generated by the Gradle 'init' task. - * - * The settings file is used to specify which projects to include in your build. - * - * Detailed information about configuring a multi-project build in Gradle can be found - * in the user manual at https://docs.gradle.org/7.4.1/userguide/multi_project_builds.html - */ - -rootProject.name = 'JavaConnector' diff --git a/v2/examples/source_connector/java/src/main/java/connector/JavaConnector.java b/v2/examples/source_connector/java/src/main/java/connector/JavaConnector.java deleted file mode 100644 index ceb4297..0000000 --- a/v2/examples/source_connector/java/src/main/java/connector/JavaConnector.java +++ /dev/null @@ -1,22 +0,0 @@ -package connector; - -import io.grpc.*; - -import java.io.IOException; - -/** - * Example Plugin Connector (gRPC server) - * In production, it will be stored as a container image - */ -public class JavaConnector { - - public static void main(String[] args) throws InterruptedException, IOException { - Server server = ServerBuilder - .forPort(50051) - .addService(new ConnectorServiceImpl()).build(); - - server.start(); - System.out.println("gRPC server started"); - server.awaitTermination(); - } -} diff --git a/v2/examples/source_connector/java/src/main/java/connector/State.java b/v2/examples/source_connector/java/src/main/java/connector/State.java deleted file mode 100644 index a9662eb..0000000 --- a/v2/examples/source_connector/java/src/main/java/connector/State.java +++ /dev/null @@ -1,5 +0,0 @@ -package connector; - -public class State { - public Integer cursor = 0; -} diff --git a/v2/examples/source_connector/nodejs/README.md b/v2/examples/source_connector/nodejs/README.md deleted file mode 100644 index 68ea2f6..0000000 --- a/v2/examples/source_connector/nodejs/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# NodeJS Connector Example - -## Steps -- Run the build.sh file. This script automates the project's build and packaging process by first installing the necessary npm packages for building and running the project. It then copies the proto files to a folder inside the Node.js project, compiles and bundles the source code into a distributable format, and finally creates a Node.js SEA (Single Executable Application) that can be executed on any system without requiring Node.js to be installed. -```commandline -sh build.sh -``` - -- Execute the binary file created to run the connector -```commandline -./binary -``` \ No newline at end of file diff --git a/v2/examples/source_connector/nodejs/build.sh b/v2/examples/source_connector/nodejs/build.sh deleted file mode 100644 index 73f6ec8..0000000 --- a/v2/examples/source_connector/nodejs/build.sh +++ /dev/null @@ -1,50 +0,0 @@ -# Install the necessary npm packages. -# `npm install` installs dependencies listed in package.json. -npm install - -# Change directory to the 'src' folder where source code is located. -cd src - -# Create a 'protos' directory if it doesn't already exist. -# This directory will be used to store protocol buffer (.proto) files. -mkdir -p protos - -# Copy all .proto files from the parent directory (5 levels up) to the 'protos' directory. -# These files are used for defining the gRPC services and messages. -cp ../../../../*.proto protos/ - -# Return to the previous directory (the project root). -cd .. - -# Run the build script defined in package.json. -# This command bundles the source code. -npm run build - -# Create a configuration file building a blob that can be injected into the single executable application -echo '{ "main": "bundle.js", "output": "sea-prep.blob" }' > sea-config.json - -# Generate the blob to be injected -node --experimental-sea-config sea-config.json - -# Create a copy of the node executable named "binary" (OS dependent). -cp $(command -v node) binary - -# Remove the signature of the binary -codesign --remove-signature binary - -# Inject the blob into the copied binary by running postject (OS dependent) -npx postject binary NODE_SEA_BLOB sea-prep.blob \ - --sentinel-fuse NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2 \ - --macho-segment-name NODE_SEA - -# Re-sign the `binary` file with the default identity. -# This step finalizes the binary by signing it for execution (OS dependent). -codesign --sign - binary - -# Remove unnecessary files -rm sea-config.json -rm sea-prep.blob -rm bundle.js - -# Please note that all commands in this file are specific to MacOS. -# For instructions on creating executables for other operating systems, please see: https://nodejs.org/api/single-executable-applications.html#single-executable-applications \ No newline at end of file diff --git a/v2/examples/source_connector/nodejs/package.json b/v2/examples/source_connector/nodejs/package.json deleted file mode 100644 index 6f93bb6..0000000 --- a/v2/examples/source_connector/nodejs/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "nodejs", - "version": "1.0.0", - "description": "", - "start": "node src/index.js", - "main": "index.js", - "scripts": { - "build": "esbuild src/index.js --bundle --platform=node --outfile=bundle.js", - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "@grpc/grpc-js": "^1.11.1", - "esbuild": "^0.23.0", - "path": "^0.12.7", - "stream": "^0.0.3" - } -} diff --git a/v2/examples/source_connector/python/README.md b/v2/examples/source_connector/python/README.md deleted file mode 100644 index f6f9132..0000000 --- a/v2/examples/source_connector/python/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Python Connector Example - -## Pre-requisites -- Python 3.9 or later - -## Steps -- Run the build.sh file to copy protos, install python dependencies in virtual environment -```commandline -sh build.sh -``` - -- Execute `run.sh` to run the connector -```commandline -sh run.sh -``` \ No newline at end of file diff --git a/v2/examples/source_connector/python/build.sh b/v2/examples/source_connector/python/build.sh deleted file mode 100755 index 471ab46..0000000 --- a/v2/examples/source_connector/python/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Creates virtual environment to install the packages and to run the connector. -python3 -m venv connector_run -source connector_run/bin/activate -# install the added packages -pip install -r requirements.txt - -# copying protos present in the root of directory to `protos` folder -mkdir -p protos -cp ../../../*.proto protos/ -# Generates the required gRPC Python files using protos into `sdk_pb2` folder -mkdir -p sdk_pb2 -python -m grpc_tools.protoc \ - --proto_path=./protos/ \ - --python_out=sdk_pb2 \ - --pyi_out=sdk_pb2 \ - --grpc_python_out=sdk_pb2 protos/*.proto -deactivate \ No newline at end of file diff --git a/v2/examples/source_connector/python/requirements.txt b/v2/examples/source_connector/python/requirements.txt deleted file mode 100644 index 4a290c6..0000000 --- a/v2/examples/source_connector/python/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -grpcio==1.60.1 -grpcio-tools==1.60.1 diff --git a/v2/examples/source_connector/python/run.sh b/v2/examples/source_connector/python/run.sh deleted file mode 100755 index 04fd809..0000000 --- a/v2/examples/source_connector/python/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -# activate the created virtual environment during the build and run the main file. -source connector_run/bin/activate -python main.py -deactivate diff --git a/v2/source_connector_sdk_v2.proto b/v2/source_connector_sdk_v2.proto deleted file mode 100644 index dd1a307..0000000 --- a/v2/source_connector_sdk_v2.proto +++ /dev/null @@ -1,93 +0,0 @@ -syntax = "proto3"; -option optimize_for = SPEED; -option java_multiple_files = true; -option go_package = "fivetran.com/fivetran_sdk_v2"; -package fivetran_sdk.v2; - -import "common_v2.proto"; - -// Fivetran (grpc client) <> SourceConnector (grpc server) -service SourceConnector { - rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} - rpc Test (TestRequest) returns (TestResponse) {} - rpc Schema (SchemaRequest) returns (SchemaResponse) {} - rpc Update (UpdateRequest) returns (stream UpdateResponse) {} -} - -message SchemaRequest { - map configuration = 1; -} - -message SchemaResponse { - oneof response { - bool schema_response_not_supported = 1; - SchemaList with_schema = 2; - TableList without_schema = 3; - } - optional bool selection_not_supported = 4; -} - -message UpdateRequest { - map configuration = 1; - optional Selection selection = 2; - optional string state_json = 3; -} - -message Selection { - oneof selection { - TablesWithNoSchema without_schema = 1; - TablesWithSchema with_schema = 2; - } -} - -message TablesWithNoSchema { - repeated TableSelection tables = 1; - bool include_new_tables = 2; -} - -message TablesWithSchema { - repeated SchemaSelection schemas = 1; - bool include_new_schemas = 2; -} - -message SchemaSelection { - bool included = 1; - string schema_name = 2; - repeated TableSelection tables = 3; - bool include_new_tables = 4; -} - -message TableSelection { - bool included = 1; - string table_name = 2; - map columns = 3; - bool include_new_columns = 4; -} - -message UpdateResponse { - oneof operation { - Record record = 1; - SchemaChange schema_change = 2; - Checkpoint checkpoint = 3; - Warning warning = 4; - Task task = 5; - } -} - -message SchemaChange { - oneof change { - SchemaList with_schema = 1; - TableList without_schema = 2; - } -} - -message Record { - optional string schema_name = 1; - string table_name = 2; - RecordType type = 3; - map data = 4; -} - -message Checkpoint { - string state_json = 1; -}