Merge pull request #1 from jerryshao/issue-2457

Issue 2457

.gitattributes

@@ -0,0 +1 @@
+text eol=lf

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,9 +1,23 @@
 name: Bug Report
 title: "[Bug report] "
 description: A bug report issue
-labels: [ "kind:bug" ]
+labels: [ "bug" ]
 body:
 
+  - type: dropdown
+    id: version
+    attributes:
+      label: Version
+      description: What version of Gravitino are you running?
+      options:
+        - main branch
+        - 0.3.1
+        - 0.3.0
+        - 0.2.0
+      default: 0
+    validations:
+      required: true
+
   - type: textarea
     attributes:
       label: Describe what's wrong

.github/ISSUE_TEMPLATE/epic.yml

@@ -1,7 +1,7 @@
 name: Epic
 title: "[EPIC] "
 description: An epic issue with multiple sub-tasks
-labels: [ "kind:epic,priority:major" ]
+labels: [ "epic" ]
 body:
 
   - type: textarea

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,7 +1,7 @@
 name: Feature Request
 title: "[FEATURE] "
 description: Suggest an idea for Gravitino
-labels: [ "kind:feature,priority:major" ]
+labels: [ "feature" ]
 body:
 
   - type: textarea

.github/ISSUE_TEMPLATE/subtask.yml

@@ -1,7 +1,7 @@
 name: Subtask
 title: "[Subtask] "
 description: A subtask issue
-labels: [ "kind:subtask" ]
+labels: [ "subtask" ]
 body:
 
   - type: textarea

.github/workflows/auto-cherry-pick.yml

@@ -7,56 +7,62 @@ on:
     types: ["closed"]
 
 jobs:
-  cherry_pick_branch_0_1:
+  cherry_pick_branch_0_3:
     runs-on: ubuntu-latest
-    name: Cherry pick into branch_0.1
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.1') && github.event.pull_request.merged == true }}
+    name: Cherry pick into branch_0.3
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.3') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Cherry pick into branch-0.1
+      - name: Cherry pick into branch-0.3
         uses: carloscastrojumo/[email protected]
         with:
-          branch: branch-0.1
+          token: ${{ secrets.BOT_TOKEN }}
+          branch: branch-0.3
           labels: |
             cherry-pick
           reviewers: |
             jerryshao
-  cherry_pick_branch_0_2:
+
+  cherry_pick_branch_0_4:
     runs-on: ubuntu-latest
-    name: Cherry pick into branch_0.2
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.2') && github.event.pull_request.merged == true }}
+    name: Cherry pick into branch_0.4
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.4') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Cherry pick into branch-0.2
+      - name: Cherry pick into branch-0.4
         uses: carloscastrojumo/[email protected]
         with:
-          branch: branch-0.2
+          token: ${{ secrets.BOT_TOKEN }}
+          branch: branch-0.4
           labels: |
             cherry-pick
           reviewers: |
             jerryshao
-  cherry_pick_branch_0_3:
+
+  cherry_pick_branch_0_5:
     runs-on: ubuntu-latest
-    name: Cherry pick into branch_0.3
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.3') && github.event.pull_request.merged == true }}
+    name: Cherry pick into branch_0.5
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.5') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Cherry pick into branch-0.3
+      - name: Cherry pick into branch-0.5
         uses: carloscastrojumo/[email protected]
         with:
-          branch: branch-0.3
+          token: ${{ secrets.BOT_TOKEN }}
+          branch: branch-0.5
           labels: |
             cherry-pick
           reviewers: |
             jerryshao
+
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build.yml

@@ -37,6 +37,7 @@ jobs:
               - meta/**
               - server/**
               - server-common/**
+              - spark-connector/**
               - trino-connector/**
               - web/**
               - docs/open-api/**
@@ -47,10 +48,27 @@ jobs:
     outputs:
       source_changes: ${{ steps.filter.outputs.source_changes }}
 
-  # This workflow contains a single job called "build"
+  compile-check:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.source_changes != 'true'
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          java-version: 8
+          distribution: 'temurin'
+
+      - name: Build with Gradle
+        run: ./gradlew build -x test -PjdkVersion=8
+
   build:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ 8, 11, 17 ]
     timeout-minutes: 30
     needs: changes
     if: needs.changes.outputs.source_changes == 'true'
@@ -61,11 +79,11 @@ jobs:
 
       - uses: actions/setup-java@v3
         with:
-          java-version: '8'
+          java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
 
       - name: Build with Gradle
-        run: ./gradlew build -PskipITs
+        run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }}
 
       - name: Upload unit tests report
         uses: actions/upload-artifact@v3

.github/workflows/docker-image.yml

@@ -12,6 +12,7 @@ on:
           - 'gravitino'
           - 'gravitino-ci-hive'
           - 'gravitino-ci-trino'
+          - 'gravitino-ci-doris'
           - 'trino'
           - 'hive'
       tag:
@@ -39,6 +40,9 @@ jobs:
           elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-trino" ]; then
             echo "image_type=trino" >> $GITHUB_ENV
             echo "image_name=datastrato/gravitino-ci-trino" >> $GITHUB_ENV
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-doris" ]; then
+            echo "image_type=doris" >> $GITHUB_ENV
+            echo "image_name=datastrato/gravitino-ci-doris" >> $GITHUB_ENV
           elif [ "${{ github.event.inputs.image }}" == "gravitino" ]; then
             echo "image_type=gravitino" >> $GITHUB_ENV
             echo "image_name=datastrato/gravitino" >> $GITHUB_ENV
@@ -76,9 +80,5 @@ jobs:
           java-version: '8'
           distribution: 'temurin'
 
-      - name: Build and Push the main branch Docker image
-        if: ${{ github.ref_name == 'main' }}
-        run: ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.tag }} --latest
-      - name: Build and Push the other branch Docker image
-        if: ${{ github.ref_name != 'main' }}
-        run: ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.tag }}
+      - name: Build and Push the Docker image
+        run: sudo rm -rf /usr/local/lib/android && sudo rm -rf /opt/hostedtoolcache/CodeQL && ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.tag }} --latest

.github/workflows/integration-test.yml

@@ -36,6 +36,7 @@ jobs:
               - meta/**
               - server/**
               - server-common/**
+              - spark-connector/**
               - trino-connector/**
               - web/**
               - docs/open-api/**
@@ -55,6 +56,8 @@ jobs:
     strategy:
       matrix:
         architecture: [linux/amd64]
+        java-version: [ 8, 11, 17 ]
+        test-mode: [ embedded, deploy ]
     env:
       DOCKER_RUN_NAME: hive-amd64
       PLATFORM: ${{ matrix.architecture }}
@@ -63,16 +66,16 @@ jobs:
 
       - uses: actions/setup-java@v3
         with:
-          java-version: '8'
+          java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
       - name: Package Gravitino
         run: |
-          ./gradlew build -x test
-          ./gradlew compileDistribution -x test
+          ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }}
+          ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }}
 
       - name: Setup debug Github Action
         if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
@@ -81,16 +84,16 @@ jobs:
       - name: Integration Test
         id: integrationTest
         run: |
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=embedded
-          ./gradlew test --rerun-tasks -PskipTests -PtestMode=deploy
+          ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }}
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
         if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
         with:
-          name: integrate test reports 
+          name: integrate test reports
           path: |
             build/reports
             integration-test/build/integration-test.log
             distribution/package/logs/gravitino-server.out
             distribution/package/logs/gravitino-server.log
+            catalogs/**/*.log

.gitignore

@@ -29,9 +29,10 @@ replay_pid*
 **/build
 gen
 **/.DS_Store
-*.iml
+**/*.iml
 out/**
 *.iws
+*.ipr
 
 distribution
 server/src/main/resources/project.properties
@@ -44,4 +45,8 @@ dev/docker/tools/docker-connector
 include web/.gitignore
 
 # node env
-.node
+.node
+
+# integration test
+integration-test/metastore_db
+catalogs/catalog-lakehouse-iceberg/spark-warehouse

CONTRIBUTING.md

@@ -17,6 +17,7 @@ Before you get started, please read and follow these guidelines to ensure a smoo
   - [Code of Conduct](#code-of-conduct)
   - [Reporting Bugs](#reporting-bugs)
   - [Suggesting Enhancements](#suggesting-enhancements)
+  - [Good First Issues](#good-first-issues)
   - [Working on Issues](#working-on-issues)
   - [Creating Pull Requests](#creating-pull-requests)
 - [Development Setup](#development-setup)
@@ -32,7 +33,7 @@ Before you get started, please read and follow these guidelines to ensure a smoo
 Either click the "Fork" button at the top right of the repository's page on GitHub OR create a fork on your local machine using `git clone`.
 
 ```bash
-git clone https://github.com/datastarto/gravitino.git
+git clone https://github.com/datastrato/gravitino.git
 cd gravitino
 ```
 
@@ -52,9 +53,13 @@ If you find a bug in Gravitino, please open an issue on GitHub. Be sure to inclu
 
 If you have ideas for enhancements or new features, feel free to create an issue to discuss them. Gravitino welcomes suggestions and provides prompt feedback on their feasibility and relevance.
 
+### Good first issues
+
+If you are new to open source or can't find something to work on check out the [Good First Issues list](https://github.com/datastrato/gravitino/contribute).
+
 ### Working on issues
 
-Check out the list of open issues and find one that interests you. You can also assign an issue to yourself to indicate that you're working on it. Please keep the issue updated with your progress.
+Check out the list of open issues and find one that interests you. You can also comment on an issue to indicate that you're working on it. Please keep the issue updated with your progress.
 
 ## Creating pull requests
 
@@ -80,15 +85,15 @@ After you have pushed your changes, create a pull request (PR) in the Gravitino
 
 ## Development setup
 
-Once you have cloned the [GitHub repository](https://github.com/datastrato/gravitino), see [how to build](/docs/how-to-build) for instructions on how to build, or you can use the provided docker images at [Datastrato's DockerHub repository](https://hub.docker.com/u/datastrato).
+Once you have cloned the [GitHub repository](https://github.com/datastrato/gravitino), see [how to build](/docs/how-to-build.md) for instructions on how to build, or you can use the provided docker images at [Datastrato's DockerHub repository](https://hub.docker.com/u/datastrato).
 
-To stop and start a local Gravitino server via ``bin/gravitino.sh start`` and ``bin/gravitino.sh stop`` in a Gravitino distribution, see [how to build](/docs/how-to-build) for more instructions.
+To stop and start a local Gravitino server via ``bin/gravitino.sh start`` and ``bin/gravitino.sh stop`` in a Gravitino distribution, see [how to build](/docs/how-to-build.md) for more instructions.
 
 ## Testing
 
 The CI infrastructure runs unit and integration tests on each pull request, please make sure these tests pass before making a pull request.
 
-The unit tests run on every build and integration tests run as needed. See [how to test](docs/how-to-test) for more information.
+The unit tests run on every build and integration tests run as needed. See [how to test](docs/how-to-test.md) for more information.
 
 When adding new code or fixing a bug be sure to add unit tests to provide coverage.
 
@@ -104,6 +109,8 @@ All files must have a license header and the build fails if any files are missin
 
 For any bugs or new code please add unit tests to provide coverage of the code. The project may not accept code without unit tests.
 
+All text files should use macOS/unix style line endings (LF) not windows style line endings (CRLF).
+
 ## Community and communication
 
 Join the [community discourse group](https://gravitino.discourse.group) to discuss ideas and seek help. You are also encouraged to use GitHub discussions and follow Datastrato on social media to stay updated on project news.

GOVERNANCE.md

@@ -5,7 +5,7 @@
 
 # Governance Policy
 
-This document provides the governance policy for the project. Maintainers agree to this policy and to follow all Project polices by adding their name to the [maintainers.md file](./MAINTAINERS.md).
+This document provides the governance policy for the project. Maintainers agree to this policy and to follow all project polices by adding their name to the [maintainers.md file](./MAINTAINERS.md).
 
 ## 1. Roles
 
@@ -25,7 +25,7 @@ This project includes the following roles.
 
 **3.2. Balance**. The development process should balance the interests of all stakeholders.
 
-**3.3. Harmonization**. Good faith efforts shall be made to resolve any conflicts.
+**3.3. Harmonization**. Good-faith efforts shall be made to resolve any conflicts.
 
 ## 4. Trademarks