Merge pull request #766 from quadratichq/next-connections

feat: Data Connectors (SQL)
quadratichq · Jul 18, 2024 · 2380f61 · 2380f61
2 parents f3fe723 + ba01746
commit 2380f61
Show file tree

Hide file tree

Showing 322 changed files with 18,972 additions and 6,703 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,6 +13,7 @@ on:
 jobs:
   test_rust:
     runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -98,16 +99,26 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+      - name: Test quadratic-connection
+        env:
+          LLVM_PROFILE_FILE: grcov-%p-%m.profraw
+          RUSTFLAGS: -Cinstrument-coverage
+          RUSTC_BOOTSTRAP: 1
+        run: |
+          cd quadratic-connection
+          npm run test:docker:ci
+
   test_unit:
     runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 18
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.11.3'
+          python-version: "3.11.3"
 
       - name: Set up Rust
         uses: moonrepo/setup-rust@v1
@@ -135,6 +146,7 @@ jobs:
 
   test_python:
     runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -152,6 +164,7 @@ jobs:
 
   test_api:
     runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -166,6 +179,7 @@ jobs:
 
   lint_rust:
     runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - name: Set up Rust
@@ -181,6 +195,7 @@ jobs:
 
   lint:
     runs-on: ubuntu-latest-8-cores
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4

diff --git a/.github/workflows/preview-branches.yml b/.github/workflows/preview-branches.yml
@@ -19,6 +19,7 @@ jobs:
     needs:
      - files_preview
      - multiplayer_preview
+     - connection_preview
     name: Create or Update Infrastructure with Pulumi
     runs-on: ubuntu-latest
     steps:
@@ -52,6 +53,7 @@ jobs:
 
         pulumi config set multiplayer-subdomain multiplayer-pr-${{ env.PR_ID }}
         pulumi config set files-subdomain files-pr-${{ env.PR_ID }}
+        pulumi config set connection-subdomain connection-pr-${{ env.PR_ID }}
         pulumi config set docker-image-tag pr-${{ env.PR_ID }}
         pulumi config set quadratic-api-uri https://quadratic-api-dev-pr-${{ env.PR_ID }}.herokuapp.com
         pulumi config set is-preview true
@@ -121,6 +123,33 @@ jobs:
         docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f quadratic-files/Dockerfile .
         docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
 
+  connection_preview:
+    runs-on: ubuntu-latest-8-cores 
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
+        aws-region: ${{ secrets.AWS_REGION }}
+
+    - name: Login to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+
+    - name: Build, Tag, and Push Image to Amazon ECR
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        ECR_REPOSITORY: quadratic-connection-development
+        IMAGE_TAG: pr-${{ github.event.pull_request.number }}
+      run: |
+        docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f quadratic-connection/Dockerfile .
+        docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
 
   # client_preview:
   #   needs: infra

diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml
@@ -116,3 +116,31 @@ jobs:
       run: |
         docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f quadratic-files/Dockerfile .
         docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
+  connection_production:
+    runs-on: ubuntu-latest-8-cores 
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_REGION }}
+
+    - name: Login to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+
+    - name: Build, Tag, and Push Image to Amazon ECR
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        ECR_REPOSITORY: quadratic-connection-production
+        IMAGE_TAG: ${{ github.sha }}
+      run: |
+        docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f quadratic-connection/Dockerfile .
+        docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
diff --git a/.gitignore b/.gitignore
@@ -36,6 +36,7 @@ venv/*
 
 # Generated Rust files
 /target
+quadratic-connection/target/
 quadratic-core/target/
 quadratic-core/tmp.txt
 quadratic-files/target/
@@ -49,19 +50,21 @@ quadratic-api/.env.local
 quadratic-api/dist
 
 # Code Coverage
+quadratic-connection/coverage
 quadratic-core/coverage
+quadratic-files/coverage
+quadratic-multiplayer/coverage
 
-# Python deployment files
+# Python Deployment files
 quadratic-kernels/python-wasm/dist/
 quadratic-client/public/*.whl
 
-quadratic-files/coverage
-quadratic-multiplayer/coverage
-
-# Docker Data
+# Docker Data Volumes
 docker/localstack/data
 docker/redis/data
 docker/postgres/data
+docker/postgres-connection/data
+docker/mysql-connection/data
 
 # JMeter
 jmeter.log
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -59,6 +59,7 @@
     "./quadratic-rust-client/Cargo.toml"
   ],
   "rust-analyzer.checkOnSave": true,
+  "rust-analyzer.cargo.unsetTest": true,
   // "rust-analyzer.checkOnSave.command": "clippy",
   "files.associations": {
     "*.grid": "json"
@@ -73,5 +74,8 @@
   "[typescript]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true
+  },
+  "[typescriptreact]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
   }
 }
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -18,144 +18,7 @@ Check out our open roles ⟶ [careers.quadratichq.com](https://careers.quadratic
 
 ### Local development environment
 
-1. Install [nvm](https://github.com/nvm-sh/nvm)
-
-2. Install Node and NPM
-
-   ```sh
-    nvm install && nvm use
-   ```
-
-3. Install [rustup](https://www.rust-lang.org/tools/install)
-
-4. Install [wasm-pack](https://rustwasm.github.io/wasm-pack/installer/)
-
-5. Install WASM toolchain
-
-   ```sh
-    rustup target add wasm32-unknown-unknown
-   ```
-
-6. Install cargo watch
-
-   ```sh
-    cargo install cargo-watch
-   ```
-
-7. Install [Python](https://wiki.python.org/moin/BeginnersGuide/Download)
-
-8. Install [Docker Desktop](https://docs.docker.com/desktop/)
-
-### Configure Auth0 account (Optional)
-
-This is only required if the `quadratic-community` Auth0 account does not work.
-
-1. Create an account on [Auth0](https://auth0.com/)
-
-2. Create a new Regular Web Application
-
-3. In the settings tab, configure the following:
-   - Allowed Callback URLs: `http://localhost:3000/login-result?redirectTo`
-   - Allowed Logout URLs: `http://localhost:3000`
-   - Allowed Web Origins: `http://localhost:3000`
-   - Allowed Origins (CORS): `http://localhost:3000`
-
-### Configure .env files
-
-1. Create .env files using the template .env.example files
-
-   ```sh
-    cp .env.example .env && \
-    cp quadratic-api/.env.example quadratic-api/.env && \
-    cp quadratic-client/.env.example quadratic-client/.env && \
-    cp quadratic-files/.env.example quadratic-files/.env && \
-    cp quadratic-multiplayer/.env.example quadratic-multiplayer/.env
-   ```
-
-   These are prefilled with `quadratic-community` Auth0 account credentials and values required to access services in docker.
-
-2. If the `quadratic-community` Auth0 account does not work, update the Auth0 values in the following files with your Auth0 account values:
-
-   - `.env`
-   - `quadratic-api/.env`
-   - `quadratic-client/.env`
-   - `quadratic-multiplayer/.env`
-
-## Run Quadratic
-
-1. Install dependencies
-
-   ```sh
-    npm install
-   ```
-
-2. Start Redis, Postgres and Localstack in docker
-
-   ```sh
-    npm run docker:up
-   ```
-
-3. Start all quadratic packages
-
-   ```sh
-    npm run start
-   ```
-
-   Press `h` in termial to open help menu. Use shortcuts from this menu to toggle watch mode, logs, etc.
-
-   To run all packages in watch mode, use
-
-   ```sh
-    npm run dev
-   ```
-
-## Run tests
-
-### TypeScript
-
-1. Go to the `quadratic-client` directory
-
-   ```sh
-    cd quadratic-client
-   ```
-
-2. Compile the Rust code
-
-   ```sh
-    build:wasm:nodejs
-   ```
-
-3. Install dependencies (run again when updating Rust)
-
-   ```sh
-    npm install
-   ```
-
-4. Run all tests
-
-   ```sh
-    npm run test:all
-   ```
-
-   or run just unit tests
-
-   ```sh
-    npm run test:unit
-   ```
-
-### Rust
-
-1. Go to the `quadratic-core` directory
-
-   ```sh
-    cd quadratic-core
-   ```
-
-2. Run test
-
-   ```sh
-    cargo test --workspace
-   ```
+See [DEVELOPMENT.md](/DEVELOPMENT.md)
 
 ## Feature requests and bugs