[pull] master from mage-ai:master

.flake8

@@ -1,4 +1,10 @@
 [flake8]
-ignore = E121,E126,W503,W504
-exclude = mage_ai/data_preparation/templates,mage_ai/orchestration/db/migrations/versions
+ignore = E121,E126,W503,W504,E203,W605,B006
+exclude =
+    mage_ai/data_preparation/templates,
+    mage_ai/orchestration/db/migrations/versions,
+    mage_integrations/mage_integrations/sources/*/tap_*/,
+    mage_integrations/mage_integrations/destinations/*/target_*/
 max-line-length = 100
+per-file-ignores =
+    mage_ai/orchestration/pipeline_scheduler.py: F401,F403

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,46 @@
+name: Bug report
+description: Create a report to help us improve
+title: '[BUG] '
+labels: ['bug']
+assignees: []
+body:
+  - type: input
+    attributes:
+      label: Mage version
+      description: What Mage version are you using?
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: To reproduce
+      description: Steps to reproduce the behavior
+      placeholder: |
+        1. Go to '...'
+        2. Click on '....'
+        3. Scroll down to '....'
+        4. See error
+  - type: textarea
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen.
+  - type: textarea
+    attributes:
+      label: Screenshots
+      description: If applicable, add screenshots to help explain your problem.
+  - type: textarea
+    attributes:
+      label: Operating system
+      placeholder: |
+        - OS: [e.g. iOS]
+        - Browser [e.g. chrome, safari]
+        - Version [e.g. 22]
+  - type: textarea
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.

.github/pull_request_template.md

@@ -20,7 +20,6 @@ Provide instructions so we can reproduce.
 - [ ] I have added unit tests that prove my fix is effective or that my feature works
 - [ ] I have commented my code, particularly in hard-to-understand areas
 - [ ] I have made corresponding changes to the documentation
-- [ ] If new documentation has been added, relative paths have been added to the appropriate section of `docs/mint.json`
 
 cc:
 <!-- Optionally mention someone to let them know about this pull request -->

.github/workflows/build_and_test.yml

@@ -16,6 +16,9 @@ on:
       - "requirements.txt"
       - "setup.py"
 
+env:
+  ENV: test_mage
+
 jobs:
   check-code-quality:
 
@@ -50,7 +53,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     needs: check-code-quality
     steps:
       - uses: actions/checkout@v3
@@ -62,8 +65,9 @@ jobs:
         run: |
           sudo apt-get install freetds-dev
           python -m pip install --upgrade pip
+          pip install "git+https://github.com/mage-ai/sqlglot#egg=sqlglot"
           pip install -r requirements.txt
-          pip install -r mage_integrations/requirements.txt
+          pip install mage_integrations/
           pip install "git+https://github.com/mage-ai/singer-python.git#egg=singer-python"
       - name: Run unit tests
         run: |
@@ -82,8 +86,7 @@ jobs:
         if: matrix.python-version == '3.10'
       - name: Create Mage test project
         run: |
-          python mage_ai/cli/main.py init test_project &&
-          echo 'help_improve_mage: false' >> test_project/metadata.yaml
+          python mage_ai/cli/main.py init test_project
         env:
           PYTHONPATH: ${{ env.pythonLocation }}:.
         if: matrix.python-version == '3.10'
@@ -103,3 +106,42 @@ jobs:
           name: playwright-report
           path: mage_ai/frontend/playwright-report/
           retention-days: 3
+
+  test_web_server_windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    needs: check-code-quality
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      # Run Playwright web UI tests only run once per workflow (Python 3.10).
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20.15.1"
+      - name: Create Mage test project
+        run: |
+          Start-Process -Wait -FilePath "C:\Program Files\Git\unins000.exe" -ArgumentList "/SILENT"
+          python -m venv venv3
+          venv3/Scripts/Activate.ps1
+          $env:PYTHONPATH = "${env:PYTHONPATH};${pwd}"
+          python -m pip install -r requirements.txt
+          python mage_ai/cli/main.py init test_project
+      - name: Build frontend, start server, and run Playwright tests
+        run: |
+          yarn install &&
+          yarn export_prod &&
+          yarn playwright install chromium &&
+          yarn playwright test -c playwright-windows.config.ci.ts
+        working-directory: mage_ai/frontend/
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: mage_ai/frontend/playwright-report/
+          retention-days: 3

.github/workflows/export_frontend.yml

@@ -0,0 +1,63 @@
+name: Export Frontend
+
+on:
+  workflow_dispatch:
+
+jobs:
+  export-frontend:
+    runs-on: ubuntu-latest
+    env:
+      NODE_VERSION: '20.15.1'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Change directory to frontend
+        working-directory: mage_ai/frontend
+        run: echo "Entering frontend directory"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Configure Git
+        run: |
+          git config user.email "[email protected]"
+          git config user.name "GitHub Actions"
+
+      - name: Fetch and Checkout to Current Branch
+        run: |
+          git fetch origin
+          git checkout ${{ github.ref_name }}
+
+      - name: Build and Export
+        run: |
+          yarn install
+          yarn export_prod
+        working-directory: mage_ai/frontend/
+
+      - name: Build and Export Base Path Version
+        run: |
+          mv next_base_path.config.js next.config.js
+          yarn install
+          yarn export_prod_base_path
+        working-directory: mage_ai/frontend/
+
+      - name: Push changes to new branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git checkout -b update-frontend-${{ github.run_id }}
+          git add mage_ai/server/frontend_dist
+          git add mage_ai/server/frontend_dist_base_path_template
+          git commit -m "[CI] Update frontend build"
+          git push origin update-frontend-${{ github.run_id }}
+
+      - name: Create Pull Request
+        run: |
+          gh pr create -B master -H update-frontend-${{ github.run_id }} \
+            --title '[CI] Update frontend build' \
+            --body 'This PR updates the frontend build artifacts.'
+        env:
+            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish_docker_image.yml

@@ -3,10 +3,14 @@ name: Build and publish Docker image
 on:
   workflow_dispatch:
     inputs:
-      branch:
-        description: 'Git branch'
+      dockerfile_branch:
+        description: 'Dockerfile branch'
         required: true
         default: master
+      feature_branch:
+        description: 'Feature branch'
+        required: false
+        default: null
       name:
         description: 'Docker image name'
         required: true
@@ -24,7 +28,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          ref: ${{ github.event.inputs.branch }}
+          ref: ${{ github.event.inputs.dockerfile_branch }}
 
       - name: Log in to Docker Hub
         uses: docker/login-action@v2
@@ -52,3 +56,5 @@ jobs:
           push: true
           tags: mageai/${{ github.event.inputs.name }}:${{ github.event.inputs.tag }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            FEATURE_BRANCH=${{ github.event.inputs.feature_branch }}

.github/workflows/publish_to_pypi.yml

@@ -29,12 +29,16 @@ jobs:
         python setup.py sdist bdist_wheel
     - name: Publish distribution to Test PyPI
       if: github.event.inputs.dry_run == 'true'
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1.8
       with:
         password: ${{ secrets.TEST_PYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
+        verbose: true
+        skip-existing: true
     - name: Publish distribution to PyPI
       if: github.event.inputs.dry_run == 'false'
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1.8
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}
+        verbose: true
+        skip-existing: true