diff --git a/.github/workflows/VeniceCI-E2ETests.yml b/.github/workflows/VeniceCI-E2ETests.yml index 1920c4f7149..12f67bf1c2d 100644 --- a/.github/workflows/VeniceCI-E2ETests.yml +++ b/.github/workflows/VeniceCI-E2ETests.yml @@ -29,11 +29,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_1 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -94,7 +94,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -126,11 +126,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_2 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -191,7 +191,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -223,11 +223,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_3 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -288,7 +288,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -320,11 +320,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_4 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -385,7 +385,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -417,11 +417,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_5 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -482,7 +482,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -514,11 +514,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_6 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -579,7 +579,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -611,11 +611,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_7 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -676,7 +676,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -708,11 +708,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_8 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -773,7 +773,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -805,11 +805,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_9 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -870,7 +870,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -902,11 +902,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_10 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -967,7 +967,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -999,11 +999,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_11 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1064,7 +1064,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1096,11 +1096,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_12 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1161,7 +1161,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1193,11 +1193,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_13 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1258,7 +1258,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1290,11 +1290,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_14 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1355,7 +1355,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1387,11 +1387,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_15 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1452,7 +1452,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1484,11 +1484,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_16 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1549,7 +1549,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1581,11 +1581,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_17 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1646,7 +1646,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1678,11 +1678,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_18 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1743,7 +1743,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1775,11 +1775,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_19 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1840,7 +1840,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1872,11 +1872,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_20 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -1937,7 +1937,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -1969,11 +1969,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_21 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -2034,7 +2034,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -2066,11 +2066,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-IntegrationTests_22 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -2131,7 +2131,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz @@ -2163,11 +2163,11 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk17-integrationTests_99 cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'temurin' @@ -2228,7 +2228,7 @@ jobs: report_paths: '**/build/test-results/test/TEST-*.xml' - name: Upload Build Artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ github.job }} path: ${{ github.job }}-jdk17-logs.tar.gz diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 00000000000..08294431efe --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,50 @@ +name: Deploy Documentation + +on: + push: + branches: [main] + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Configure Git Credentials + run: | + git remote set-head origin --auto + git remote add upstream https://github.com/linkedin/venice + git fetch upstream + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: ~/.cache + restore-keys: | + mkdocs-material- + - run: pip install -r docs/doc-requirements.txt + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Validate Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + + - name: Generate Javadoc + run: ./gradlew aggregateJavadoc + + - name: Copy Javadoc + run: | + mkdir -p docs/javadoc + cp -r build/javadoc/* docs/javadoc/ + + - run: mkdocs gh-deploy --force \ No newline at end of file diff --git a/.github/workflows/publish-javadoc.yml b/.github/workflows/publish-javadoc.yml deleted file mode 100644 index d237342df49..00000000000 --- a/.github/workflows/publish-javadoc.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Deploy Javadoc -on: - push: - branches: - - main - -permissions: - contents: write - -jobs: - PublishJavaDoc: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - shell: bash - run: | - git remote set-head origin --auto - git remote add upstream https://github.com/linkedin/venice - git fetch upstream - git config --global user.name "JavaDoc Bot" - git config --global user.email "javadoc_bot@invalid.com" - git switch -C javadoc - git rebase origin/main - git push -f origin javadoc - - name: Validate Gradle wrapper - uses: gradle/actions/wrapper-validation@v3 - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 - - name: Build Javadoc - run: ./gradlew aggregateJavadoc - - name: Deploy to GitHub Page - uses: JamesIves/github-pages-deploy-action@v4.7.6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: javadoc - clean: true - folder: "build/javadoc" - target-folder: docs/javadoc \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8a460fad761..ebc5db9d6bc 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ .shelf/ out/ build/ +site/ internal/venice-test-common/tmp/ internal/venice-test-common/dump.complete.hprof internal/venice-test-common/src/jmh/generated diff --git a/build.gradle b/build.gradle index c978f18088f..847e0f27a27 100644 --- a/build.gradle +++ b/build.gradle @@ -632,6 +632,12 @@ spotless { target '**/src/**/*.java' targetExclude '**/generated/**' } + format 'markdown', { + target '**/*.md' + targetExclude '**/build/**', '**/target/**', '**/.gradle/**' + prettier(['prettier': '2.8.8', 'prettier-plugin-java': '2.2.0']) + .config(['proseWrap': 'always', 'printWidth': 120]) + } } task setupWorkspace { diff --git a/docs/Gemfile b/docs/Gemfile deleted file mode 100644 index 1a433b013a9..00000000000 --- a/docs/Gemfile +++ /dev/null @@ -1,7 +0,0 @@ -source "https://rubygems.org" - -# do NOT include the jekyll gem ! -gem "github-pages", "~> 228", group: :jekyll_plugins -gem "kramdown-parser-gfm" -gem "webrick", "~> 1.8" -gem "csv" diff --git a/docs/README.md b/docs/README.md index 37b7fd77d2b..e64ef67a8f1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,29 +22,31 @@ Venice is a derived data storage platform, providing the following characteristics: -1. High throughput asynchronous ingestion from batch and streaming sources (e.g. [Hadoop](https://github.com/apache/hadoop) and [Samza](https://github.com/apache/samza)). +1. High throughput asynchronous ingestion from batch and streaming sources (e.g. + [Hadoop](https://github.com/apache/hadoop) and [Samza](https://github.com/apache/samza)). 2. Low latency online reads via remote queries or in-process caching. 3. Active-active replication between regions with CRDT-based conflict resolution. 4. Multi-cluster support within each region with operator-driven cluster assignment. 5. Multi-tenancy, horizontal scalability and elasticity within each cluster. -The above makes Venice particularly suitable as the stateful component backing a Feature Store, such as [Feathr](https://github.com/feathr-ai/feathr). -AI applications feed the output of their ML training jobs into Venice and then query the data for use during online -inference workloads. +The above makes Venice particularly suitable as the stateful component backing a Feature Store, such as +[Feathr](https://github.com/feathr-ai/feathr). AI applications feed the output of their ML training jobs into Venice and +then query the data for use during online inference workloads. + +## Overview -# Overview Venice is a system which straddles the offline, nearline and online worlds, as illustrated below. ![High Level Architecture Diagram](assets/images/high_level_architecture.drawio.svg) -## Dependency +### Dependency You can add a dependency on Venice to any Java project as specified below. Note that, currently, Venice dependencies are not published on Maven Central and therefore require adding an extra repository definition. All published jars can be -seen [here](https://linkedin.jfrog.io/ui/native/venice/com/linkedin/venice/). Usually, the project is released a few +seen [here](https://linkedin.jfrog.io/ui/native/venice/com/linkedin/venice/). Usually, the project is released a few times per week. -### Gradle +#### Gradle Add the following to your `build.gradle`: @@ -62,7 +64,7 @@ dependencies { } ``` -### Maven +#### Maven Add the following to your `pom.xml`: @@ -91,74 +93,77 @@ Add the following to your `pom.xml`: ``` -## APIs -From the user's perspective, Venice provides a variety of read and write APIs. These are fully decoupled from one +### APIs + +From the user's perspective, Venice provides a variety of read and write APIs. These are fully decoupled from one another, in the sense that no matter which write APIs are used, any of the read APIs are available. -Furthermore, Venice provides a rich spectrum of options in terms of simplicity on one end, and sophistication on the -other. It is easy to get started with the simpler APIs, and later on decide to enhance the use case via more advanced -APIs, either in addition to or instead of the simpler ones. In this way, Venice can accompany users as their +Furthermore, Venice provides a rich spectrum of options in terms of simplicity on one end, and sophistication on the +other. It is easy to get started with the simpler APIs, and later on decide to enhance the use case via more advanced +APIs, either in addition to or instead of the simpler ones. In this way, Venice can accompany users as their requirements evolve, in terms of scale, latency and functionality. The following diagram presents these APIs and summarizes the components coming into play to make them work. ![API Overview](assets/images/api_overview.drawio.svg) -### Write Path +#### Write Path -The Venice write path can be broken down into three granularities: full dataset swap, insertion of many rows into an +The Venice write path can be broken down into three granularities: full dataset swap, insertion of many rows into an existing dataset, and updates of some columns of some rows. All three granularities are supported by Hadoop and Samza. -In addition, any service can asynchronously produce single row inserts and updates as well, using the -[Online Producer](./user_guide/write_api/online_producer.md) library. The table below summarizes the write operations +In addition, any service can asynchronously produce single row inserts and updates as well, using the +[Online Producer](./user-guide/write-apis/online-producer.md) library. The table below summarizes the write operations supported by each platform: -| | [Hadoop](./user_guide/write_api/push_job.md) | [Samza](./user_guide/write_api/stream_processor.md) | [Any Service](./user_guide/write_api/online_producer.md) | -|-------------------------------------------------:|:--------------------------------------------:|:---------------------------------------------------:|:--------------------------------------------------------:| -| Full dataset swap | ✅ | ✅ | | -| Insertion of some rows into an existing dataset | ✅ | ✅ | ✅ | -| Updates to some columns of some rows | ✅ | ✅ | ✅ | +| | [Hadoop](./user-guide/write-apis/batch-push.md) | [Samza](./user-guide/write-apis/stream-processor.md) | [Any Service](./user-guide/write-apis/online-producer.md) | +| ----------------------------------------------: | :---------------------------------------------: | :--------------------------------------------------: | :-------------------------------------------------------: | +| Full dataset swap | ✅ | ✅ | | +| Insertion of some rows into an existing dataset | ✅ | ✅ | ✅ | +| Updates to some columns of some rows | ✅ | ✅ | ✅ | + +##### Hybrid Stores -#### Hybrid Stores -Moreover, the three granularities of write operations can all be mixed within a single dataset. A dataset which gets +Moreover, the three granularities of write operations can all be mixed within a single dataset. A dataset which gets full dataset swaps in addition to row insertion or row updates is called _hybrid_. -As part of configuring a store to be _hybrid_, an important concept is the _rewind time_, which defines how far back +As part of configuring a store to be _hybrid_, an important concept is the _rewind time_, which defines how far back should recent real-time writes be rewound and applied on top of the new generation of the dataset getting swapped in. -Leveraging this mechanism, it is possible to overlay the output of a stream processing job on top of that of a batch -job. If using partial updates, then it is possible to have some of the columns be updated in real-time and some in +Leveraging this mechanism, it is possible to overlay the output of a stream processing job on top of that of a batch +job. If using partial updates, then it is possible to have some of the columns be updated in real-time and some in batch, and these two sets of columns can either overlap or be disjoint, as desired. -#### Write Compute +##### Write Compute + Write Compute includes two kinds of operations, which can be performed on the value associated with a given key: - **Partial update**: set the content of a field within the value. -- **Collection merging**: add or remove entries in a set or map. +- **Collection merging**: add or remove entries in a set or map. -N.B.: Currently, write compute is only supported in conjunction with active-passive replication. Support for -active-active replication is under development. +N.B.: Currently, write compute is only supported in conjunction with active-passive replication. Support for +active-active replication is under development. -### Read Path +#### Read Path Venice supports the following read APIs: - **Single get**: get the value associated with a single key - **Batch get**: get the values associated with a set of keys -- **Read compute**: project some fields and/or compute some function on the fields of values associated with a set of +- **Read compute**: project some fields and/or compute some function on the fields of values associated with a set of keys. When using the read compute DSL, the following functions are currently supported: - - **Dot product**: perform a dot product on the float vector stored in a given field, against another float vector + - **Dot product**: perform a dot product on the float vector stored in a given field, against another float vector provided as query param, and return the resulting scalar. - - **Cosine similarity**: perform a cosine similarity on the float vector stored in a given field, against another + - **Cosine similarity**: perform a cosine similarity on the float vector stored in a given field, against another float vector provided as query param, and return the resulting scalar. - - **Hadamard product**: perform a Hadamard product on the float vector stored in a given field, against another float + - **Hadamard product**: perform a Hadamard product on the float vector stored in a given field, against another float vector provided as query param, and return the resulting vector. - **Collection count**: return the number of items in the collection stored in a given field. -#### Client Modes +##### Client Modes There are two main modes for accessing Venice data: -- **Classical Venice** (stateless): You can perform remote queries against Venice's distributed backend service. If +- **Classical Venice** (stateless): You can perform remote queries against Venice's distributed backend service. If using read compute operations in this mode, the queries are pushed down to the backend and only the computation results are returned to the client. There are two clients capable of such remote queries: - **Thin Client**: This is the simplest client, which sends requests to the router tier, which itself sends requests @@ -166,47 +171,60 @@ There are two main modes for accessing Venice data: - **Fast Client**: This client is partitioning-aware, and can therefore send requests directly to the correct server instance, skipping the routing tier. Note that this client is still under development and may not be as stable nor at functional parity with the Thin Client. -- **Da Vinci** (stateful): Alternatively, you can eagerly load some or all partitions of the dataset and perform queries - against the resulting local cache. Future updates to the data continue to be streamed in and applied to the local +- **Da Vinci** (stateful): Alternatively, you can eagerly load some or all partitions of the dataset and perform queries + against the resulting local cache. Future updates to the data continue to be streamed in and applied to the local cache. The table below summarizes the clients' characteristics: -| | Network Hops | Typical latency (p99) | State Footprint | -|-------------------------------:|:--------------:|:-----------------------:|:---------------------------------:| -| Thin Client | 2 | < 10 milliseconds | Stateless | -| Fast Client | 1 | < 2 milliseconds | Minimal (routing metadata only) | -| Da Vinci Client (RAM + SSD) | 0 | < 1 millisecond | Bounded RAM, full dataset on SSD | -| Da Vinci Client (all-in-RAM) | 0 | < 10 microseconds | Full dataset in RAM | +| | Network Hops | Typical latency (p99) | State Footprint | +| ---------------------------: | :----------: | :-------------------: | :------------------------------: | +| Thin Client | 2 | < 10 milliseconds | Stateless | +| Fast Client | 1 | < 2 milliseconds | Minimal (routing metadata only) | +| Da Vinci Client (RAM + SSD) | 0 | < 1 millisecond | Bounded RAM, full dataset on SSD | +| Da Vinci Client (all-in-RAM) | 0 | < 10 microseconds | Full dataset in RAM | -All of these clients share the same read APIs described above. This enables users to make changes to their +All of these clients share the same read APIs described above. This enables users to make changes to their cost/performance tradeoff without needing to rewrite their applications. -# Resources +## Resources + +The _Open Sourcing Venice_ +[blog](https://engineering.linkedin.com/blog/2022/open-sourcing-venice--linkedin-s-derived-data-platform) and +[conference talk](https://www.youtube.com/watch?v=pJeg4V3JgYo) are good starting points to get an overview of what use +cases and scale can Venice support. For more Venice posts, talks and podcasts, see our +[Learn More](./resources/learn-more.md) page. -The _Open Sourcing Venice_ [blog](https://engineering.linkedin.com/blog/2022/open-sourcing-venice--linkedin-s-derived-data-platform) -and [conference talk](https://www.youtube.com/watch?v=pJeg4V3JgYo) are good starting points to get an overview of what -use cases and scale can Venice support. For more Venice posts, talks and podcasts, see our [Learn More](./user_guide/learn_more.md) -page. +### Getting Started -## Getting Started -Refer to the [Venice quickstart](./quickstart/quickstart.md) to create your own Venice cluster and play around with some -features like creating a data store, batch push, incremental push, and single get. We recommend sticking to our latest +Refer to the [Venice quickstart](./getting-started/index.md) to create your own Venice cluster and play around with some +features like creating a data store, batch push, incremental push, and single get. We recommend sticking to our latest [stable release](https://blog.venicedb.org/stable-releases). -## Community +### Community + Feel free to engage with the community using our: + -- [](http://slack.venicedb.org) [Slack workspace](http://slack.venicedb.org) + +- [](http://slack.venicedb.org) + [Slack workspace](http://slack.venicedb.org) - Archived and publicly searchable on [Linen](http://linen.venicedb.org) -- [](https://www.linkedin.com/groups/14129519/) [LinkedIn group](https://www.linkedin.com/groups/14129519/) -- [](https://github.com/linkedin/venice/issues) [GitHub issues](https://github.com/linkedin/venice/issues) -- [](./dev_guide/how_to/how_to.md) [Contributor's guide](./dev_guide/how_to/how_to.md) +- [](https://www.linkedin.com/groups/14129519/) + [LinkedIn group](https://www.linkedin.com/groups/14129519/) +- [](https://github.com/linkedin/venice/issues) + [GitHub issues](https://github.com/linkedin/venice/issues) +- [](./contributing/index.md) + [Contributor's guide](./contributing/index.md) Follow us to hear more about the progress of the Venice project and community: -- [](https://blog.venicedb.org) [Blog](https://blog.venicedb.org) -- [](https://bsky.app/profile/venicedb.org) [Bluesky handle](https://bsky.app/profile/venicedb.org) -- [](https://www.linkedin.com/company/venicedb) [LinkedIn page](https://www.linkedin.com/company/venicedb) -- [](https://x.com/VeniceDataBase) [X handle](https://x.com/VeniceDataBase) -- [](https://youtube.com/@venicedb) [YouTube channel](https://youtube.com/@venicedb) +- [](https://blog.venicedb.org) [Blog](https://blog.venicedb.org) +- [](https://bsky.app/profile/venicedb.org) + [Bluesky handle](https://bsky.app/profile/venicedb.org) +- [](https://www.linkedin.com/company/venicedb) + [LinkedIn page](https://www.linkedin.com/company/venicedb) +- [](https://x.com/VeniceDataBase) + [X handle](https://x.com/VeniceDataBase) +- [](https://youtube.com/@venicedb) + [YouTube channel](https://youtube.com/@venicedb) diff --git a/docs/_config.yml b/docs/_config.yml deleted file mode 100644 index 79b90cf5a30..00000000000 --- a/docs/_config.yml +++ /dev/null @@ -1,23 +0,0 @@ -title: Venice -remote_theme: just-the-docs/just-the-docs -favicon_ico: "/assets/style/favicon.ico" - -color_scheme: dark - -nav_external_links: - - title: Javadoc - url: ./javadoc/index.html - hide_icon: false - -exclude: - # javadoc/legal has some markdowns that get picked up by jekyll/just-the-docs - - javadoc/legal - - vendor/ - -# Footer "Edit this page on GitHub" link text -gh_edit_link: true # show or hide edit this page link -gh_edit_link_text: "✏️ Edit this page on GitHub" -gh_edit_repository: "https://github.com/linkedin/venice" # the github URL for your repo -gh_edit_branch: "main" # the branch that your docs is served from -gh_edit_source: docs # the source that your files originate from -gh_edit_view_mode: "edit" # "tree" or "edit" if you want the user to jump into the editor immediately diff --git a/docs/_sass/custom/custom.scss b/docs/_sass/custom/custom.scss deleted file mode 100644 index 151d4b84792..00000000000 --- a/docs/_sass/custom/custom.scss +++ /dev/null @@ -1,8 +0,0 @@ -// custom SCSS (or CSS) goes here - -.main-content { - h4 { - // h4 titles are a bit flimsy, so let's embolden them a bit! - font-weight: bold; - } -} \ No newline at end of file diff --git a/docs/assets/style/HEAD_white_RGB.svg b/docs/assets/style/HEAD_white_RGB.svg new file mode 100644 index 00000000000..65e16f2cf34 --- /dev/null +++ b/docs/assets/style/HEAD_white_RGB.svg @@ -0,0 +1,13 @@ + + + + + + + + \ No newline at end of file diff --git a/docs/assets/style/extra.css b/docs/assets/style/extra.css new file mode 100644 index 00000000000..d9b27cf4227 --- /dev/null +++ b/docs/assets/style/extra.css @@ -0,0 +1,49 @@ +/* Venice Documentation Custom Styles */ +/* Brand colors: Black and White */ + +:root { + --venice-black: #000000; + --venice-white: #ffffff; +} + +[data-md-color-scheme="slate"] { + --md-primary-fg-color: var(--venice-black); + --md-accent-fg-color: var(--venice-white); +} + +[data-md-color-scheme="default"] { + --md-primary-fg-color: var(--venice-black); + --md-accent-fg-color: var(--venice-black); +} + +.md-typeset table:not([class]) { + border-radius: 0.5rem; + overflow: hidden; +} + +.md-button { + border-radius: 0.5rem; +} + +.md-typeset code { + border-radius: 0.25rem; +} + +.md-typeset pre > code { + border-radius: 0.5rem; +} + +/* Hide permalink icon by default, show only on heading hover */ +.md-typeset .headerlink { + opacity: 0; + transition: opacity 0.2s; +} + +.md-typeset h1:hover .headerlink, +.md-typeset h2:hover .headerlink, +.md-typeset h3:hover .headerlink, +.md-typeset h4:hover .headerlink, +.md-typeset h5:hover .headerlink, +.md-typeset h6:hover .headerlink { + opacity: 1; +} diff --git a/docs/dev_guide/java.md b/docs/contributing/architecture/java-internals.md similarity index 71% rename from docs/dev_guide/java.md rename to docs/contributing/architecture/java-internals.md index 3ddbd9c2539..612c7254d01 100644 --- a/docs/dev_guide/java.md +++ b/docs/contributing/architecture/java-internals.md @@ -1,33 +1,20 @@ ---- -layout: default -title: Java -parent: Developer Guides -permalink: /docs/dev_guide/java ---- # Java -{: .no_toc } -The Venice codebase is completely in Java, and therefore it is important to have at least a basic understanding of the -language and its runtime, the Java Virtual Machine (JVM). This page is not intended to be an in-depth resource on Java, -but might be a useful starting point for Java beginners. Links to more in-depth resources will be provided along the -way, and in cases where there are many distinct ways of using Java, guidance will be provided in terms of the Venice +The Venice codebase is completely in Java, and therefore it is important to have at least a basic understanding of the +language and its runtime, the Java Virtual Machine (JVM). This page is not intended to be an in-depth resource on Java, +but might be a useful starting point for Java beginners. Links to more in-depth resources will be provided along the +way, and in cases where there are many distinct ways of using Java, guidance will be provided in terms of the Venice project's expectations. -## Table of Contents -{: .no_toc } - -- TOC -{:toc} - ## Java in a Nutshell -Java as a language is statically typed, dynamically compiled, object-oriented, and provides managed memory. Let's +Java as a language is statically typed, dynamically compiled, object-oriented, and provides managed memory. Let's briefly go over each of these buzzwords. ### Statically Typed Static typing, also known as strong typing, is the notion that a variable is declared to have a given type, and that the -language provides guarantees that it is only possible to assign data to this variable which conforms to the declared +language provides guarantees that it is only possible to assign data to this variable which conforms to the declared type. This guarantee is ensured at compilation-time, which is good in the sense that compilation happens early in the development lifecycle, therefore preventing trivial mistakes from creeping into production systems and being discovered there, at run-time, when it is already too late. Static typing therefore provides a maintenance benefit @@ -37,29 +24,29 @@ which a dynamically typed language could not. This enables certain low-level opt ### Dynamically Compiled -Although Java programs need to be compiled, they are not compiled directly to machine code, but rather to an +Although Java programs need to be compiled, they are not compiled directly to machine code, but rather to an "intermediate representation" called bytecode. The bytecode can then be loaded into a Java Virtual Machine (JVM), which interprets it, and dynamically compiles architecture-specific code to run it. This means that Java code is portable ("compile once, run everywhere"), though in practice, in the case of Venice, the code has been exercised mainly on Linux (for production usage) and macOS (as a development environment). In theory, it should work on Windows or other platforms though it has not been tested there. -An additional benefit of dynamic compilation is, again, performance. The JVM features a JIT (Just-in-time) Compiler +An additional benefit of dynamic compilation is, again, performance. The JVM features a JIT (Just-in-time) Compiler which uses heuristics to analyze which code paths are most frequently called. These code paths end up being recompiled into even more optimal versions. ### Object-Oriented Java at its core is an object-oriented language, although over the years it has also added certain other paradigms such -as functional programming. What it means concretely is that design patterns such as encapsulation, inheritance and +as functional programming. What it means concretely is that design patterns such as encapsulation, inheritance and polymorphism are well-supported. ### Managed Memory An important aspect of Java is that memory management is abstracted away from the programmer. In particular, there is no need to explicitly _free_ allocated memory, and there is no risk of accidentally accessing a memory address after it has -been freed. This design improves both reliability and security, however, it comes at the cost of needing garbage -collection, which in some cases is a performance drag. These various aspects are explored more in-depth in later +been freed. This design improves both reliability and security, however, it comes at the cost of needing garbage +collection, which in some cases is a performance drag. These various aspects are explored more in-depth in later sections. ## Other Languages @@ -72,8 +59,8 @@ Furthermore, there may be other languages introduced to the Venice codebase in t ### C++ -There are two major dependencies which are written in C++, and which are accessed via the [Java Native Interface](https://docs.oracle.com/en/java/javase/17/docs/specs/jni/intro.html) -(JNI): +There are two major dependencies which are written in C++, and which are accessed via the +[Java Native Interface](https://docs.oracle.com/en/java/javase/17/docs/specs/jni/intro.html) (JNI): - The [RocksDB](https://rocksdb.org) storage engine. - The [ZSTD](https://facebook.github.io/zstd/) compression algorithm. @@ -92,8 +79,9 @@ Scala dependencies to the most minimal number of modules possible. In particular - The [Kafka](https://kafka.apache.org) broker is written in Scala, but not the Kafka client library. The broker is only used within integration tests, and the main code should not depend on it. -- The [Spark](https://spark.apache.org) framework is a runtime dependency of the [Venice Push Job](../user_guide/write_api/push_job.md), - and therefore VPJ is the only production module which carries a (transitive) dependency to Scala. +- The [Spark](https://spark.apache.org) framework is a runtime dependency of the + [Venice Push Job](../../user-guide/write-apis/batch-push.md), and therefore VPJ is the only production module which + carries a (transitive) dependency to Scala. ### Groovy @@ -110,12 +98,12 @@ there is no intent to use JS anywhere else in the codebase. ## Java Versions Currently, the Venice project intends to support Java versions 8, 11 and 17. This means that the codebase cannot use -language features that have been introduced in versions later than 8. We hope to lift this restriction in the future -and drop support for older Java versions. For the time being, the build ensures that compilation will fail if recent +language features that have been introduced in versions later than 8. We hope to lift this restriction in the future and +drop support for older Java versions. For the time being, the build ensures that compilation will fail if recent language features are used. -The main motivation for maintaining compatibility with older Java versions is to support legacy applications which -depend on the Venice client libraries. The Venice services (controller, router and server) also work on older Java +The main motivation for maintaining compatibility with older Java versions is to support legacy applications which +depend on the Venice client libraries. The Venice services (controller, router and server) also work on older Java versions though in practice we run them on Java 17 JVMs, which provide better performance. As much as possible, it is recommended to run all Venice code on the latest supported JVM version. @@ -129,49 +117,52 @@ characteristics of how memory is allocated, garbage collected, laid out, measure Memory can be allocated either on the Java _stack_, the Java _heap_ or in _native_ memory. Allocations to the stack include all local variables of primitive types (boolean, int, float, etc.) as well as pointers -to objects. [Local variables](https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html) are those +to objects. [Local variables](https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html) are those defined within the scope of a function and are therefore inaccessible outside that scope. Allocations on the heap include all objects, as well as the fields and pointers they contain. -Allocations to native memory are special in the sense that they require extra care (see Garbage Collection, below). +Allocations to native memory are special in the sense that they require extra care (see Garbage Collection, below). Within the Venice codebase, there is no explicit allocation of native memory, but some of Venice's dependencies do use native memory, for example [Netty](https://netty.io) and RocksDB. ### Garbage Collection -Garbage Collection (usually referred to as GC) is an important aspect of Java. The JVM provides "managed memory", +Garbage Collection (usually referred to as GC) is an important aspect of Java. The JVM provides "managed memory", meaning that memory is automatically allocated and deallocated as necessary, with minimal effort from the developer. Although this is ergonomically useful, it is still important to understand the basic principles of GC as it can have -performance implications, and Venice is a performance-sensitive project. Let's take the stack, heap and native +performance implications, and Venice is a performance-sensitive project. Let's take the stack, heap and native allocations introduced previously and see how each of them behaves in terms of GC. Memory allocated on the stack is automatically deallocated as soon as it goes out of scope (e.g. when the function, or if block or while loop in which it was allocated finishes). This makes stack allocations very efficient since there is not "leftover" work to take care of (see below). -Memory allocated on the heap, on the other hand, can (at least [potentially](https://blogs.oracle.com/javamagazine/post/escape-analysis-in-the-hotspot-jit-compiler)) -have a lifespan greater than the scope within which it was allocated. For example, if an allocated object is assigned to -the field of another object, then the lifecycle of the former object becomes tied to that of the latter one. These -objects on the heap eventually need to be cleaned up, which is the responsibility of the Garbage Collector and is -accomplished asynchronously via background threads, reference counting and other mechanisms. The details are outside the -scope of this wiki page, but suffice to say that GC has a cost, and so we do care about minimizing garbage generation in +Memory allocated on the heap, on the other hand, can (at least +[potentially](https://blogs.oracle.com/javamagazine/post/escape-analysis-in-the-hotspot-jit-compiler)) have a lifespan +greater than the scope within which it was allocated. For example, if an allocated object is assigned to the field of +another object, then the lifecycle of the former object becomes tied to that of the latter one. These objects on the +heap eventually need to be cleaned up, which is the responsibility of the Garbage Collector and is accomplished +asynchronously via background threads, reference counting and other mechanisms. The details are outside the scope of +this wiki page, but suffice to say that GC has a cost, and so we do care about minimizing garbage generation in performance-sensitive hot paths. Finally, native memory allocation requires manual memory management, as the JVM will not take care of automatically deallocating it as part of GC. This is the reason why it is important, for example, to close objects returned by RocksDB -when they are no longer in use, and to invoke the [ref counting APIs of Netty](https://netty.io/wiki/reference-counted-objects.html) -when appropriate. +when they are no longer in use, and to invoke the +[ref counting APIs of Netty](https://netty.io/wiki/reference-counted-objects.html) when appropriate. #### Memory Leaks Memory allocated on the heap or natively can leak, which causes severe operational issues. In the case of heap allocations, a leak can occur if references to unneeded objects are maintained. This results in GC -not being capable of eliminating this garbage since the ref counting concludes that the referenced objects are -ineligible for collection. After the heap fills up, further allocations will result in [OutOfMemoryError](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/OutOfMemoryError.html) -getting thrown. In order to debug this type of issue, it is useful to configure the JVM with the -`-XX:HeapDumpOnOutOfMemoryError` config flag, and conduct heap dump analysis. See these [mem leak troubleshooting tips](https://docs.oracle.com/en/java/javase/17/troubleshoot/troubleshooting-memory-leaks.html) +not being capable of eliminating this garbage since the ref counting concludes that the referenced objects are +ineligible for collection. After the heap fills up, further allocations will result in +[OutOfMemoryError](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/OutOfMemoryError.html) getting +thrown. In order to debug this type of issue, it is useful to configure the JVM with the +`-XX:HeapDumpOnOutOfMemoryError` config flag, and conduct heap dump analysis. See these +[mem leak troubleshooting tips](https://docs.oracle.com/en/java/javase/17/troubleshoot/troubleshooting-memory-leaks.html) to dig deeper. In the case of native allocations, a leak can occur if the Java code fails to explicitly release the objects. This can @@ -183,13 +174,14 @@ It can be useful to understand how objects are laid out in memory, in order to g structures we design. In general, the main design criteria is simply to adhere to object-oriented principles in order to achieve encapsulation and other characteristics of interest. But in cases where some pieces of data need to be allocated on the hot path (i.e., at high throughput), or be retained for long periods of time, it can be useful to pay attention -to what the heap size of the data is going to be, with an eye towards minimizing its footprint. This section is a just a -high-level overview, and more details can be found in [Java Objects Inside Out](https://shipilev.net/jvm/objects-inside-out/). +to what the heap size of the data is going to be, with an eye towards minimizing its footprint. This section is a just a +high-level overview, and more details can be found in +[Java Objects Inside Out](https://shipilev.net/jvm/objects-inside-out/). #### Memory-Impacting Configurations Note that memory layout details are dependent on which version of the JVM is used, as well as various JVM settings. In -order to facilitate explanations throughout this whole section, we will provide concrete memory size numbers based on a +order to facilitate explanations throughout this whole section, we will provide concrete memory size numbers based on a specific (though, arguably, fairly common) set of JVM configurations: - 64 bits JVM @@ -203,10 +195,10 @@ specific (though, arguably, fairly common) set of JVM configurations: #### Object Headers All objects in Java have a header, which is made up of a mark word and a class pointer. The mark word contains various -JVM internal details such as GC-related bookkeeping, the identity hashcode, and locking information, while the class +JVM internal details such as GC-related bookkeeping, the identity hashcode, and locking information, while the class pointer is a reference to the memory address of the class definition inside the metaspace. -Assuming the above JVM configuration details, the object layout will carry 12 bytes worth of header, and therefore look +Assuming the above JVM configuration details, the object layout will carry 12 bytes worth of header, and therefore look like this: ``` @@ -214,7 +206,7 @@ like this: |<---------- 8 bytes ---------->|<-- 4 bytes -->|<-- N bytes -->| ``` -In the case of arrays, there is one more header field, which is an `int` storing the length, and that is followed by +In the case of arrays, there is one more header field, which is an `int` storing the length, and that is followed by each element stored in the array (either primitives stored inline, or pointers to Objects stored elsewhere on the heap). The array, therefore, has a 16 bytes header, which looks like this: @@ -225,10 +217,10 @@ The array, therefore, has a 16 bytes header, which looks like this: #### Alignment -The previous subsection taught us that (under the example JVM settings above) an Object's header takes up 12 bytes, -however, even if this Object carried no fields at all, its actual size on the heap would still be larger than 12 bytes! -That is because objects are "aligned" in memory, and the alignment size (under default JVM settings) is 8 bytes. -Therefore, even the basic Java `Object` with no fields in it, having just a 12 bytes header, would actually take up 16 +The previous subsection taught us that (under the example JVM settings above) an Object's header takes up 12 bytes, +however, even if this Object carried no fields at all, its actual size on the heap would still be larger than 12 bytes! +That is because objects are "aligned" in memory, and the alignment size (under default JVM settings) is 8 bytes. +Therefore, even the basic Java `Object` with no fields in it, having just a 12 bytes header, would actually take up 16 bytes on the heap: ``` @@ -239,43 +231,46 @@ bytes on the heap: From this, we can derive a rule of thumb that every time we use an Object, we are spending at least 16 bytes, and if the object carries a non-trivial number of fields, then likely more than that. -More nuances on alignment and its downstream effects can be learned in [JVM Anatomy Quark #24: Object Alignment](https://shipilev.net/jvm/anatomy-quarks/24-object-alignment/). +More nuances on alignment and its downstream effects can be learned in +[JVM Anatomy Quark #24: Object Alignment](https://shipilev.net/jvm/anatomy-quarks/24-object-alignment/). #### Size of Primitives Below is the size in memory of all primitive types: | Primitive type | Size in bytes | -|----------------|---------------| +| -------------- | ------------- | | `boolean` | 1 | | `byte` | 1 | | `char` | 2 | | `short` | 2 | | `float` | 4 | | `int` | 4 | -| `double` | 8 | +| `double` | 8 | | `long` | 8 | Note that all of the above are compact representations, meaning that every single bit carries a significant piece of information, except for `boolean` which carries only one significant bit out of the 8 bits that make up its byte. -Even when part of a `boolean[]` primitive array, each element will still take 1 byte. This JVM design choice makes -certain operations more efficient for the CPU to perform, and protects against [word tearing](https://shipilev.net/blog/2014/jmm-pragmatics/#_part_ii_word_tearing), -at the cost of memory waste. In cases where the opposite tradeoffs are desired, it is possible to use a [BitSet](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/BitSet.html), -which is more space-efficient thanks to "bit packing", and provides support for bit-related operations such as `AND`, -`OR`, `XOR`, intersection, etc. +Even when part of a `boolean[]` primitive array, each element will still take 1 byte. This JVM design choice makes +certain operations more efficient for the CPU to perform, and protects against +[word tearing](https://shipilev.net/blog/2014/jmm-pragmatics/#_part_ii_word_tearing), at the cost of memory waste. In +cases where the opposite tradeoffs are desired, it is possible to use a +[BitSet](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/BitSet.html), which is more +space-efficient thanks to "bit packing", and provides support for bit-related operations such as `AND`, `OR`, `XOR`, +intersection, etc. #### Size Comparison Between Primitives and Objects For each of the primitive types, there exists an equivalent Object type. Using the Object carries the header overhead, while the primitive type does not. Continuing with the example JVM settings above, we can see that any primitive equal -or smaller to an `int` can fit "within the shadow" of the object alignment, and therefore takes up 16 bytes in Object +or smaller to an `int` can fit "within the shadow" of the object alignment, and therefore takes up 16 bytes in Object form. Whereas the `double` and `long` primitives, if packed inside of a `Double` or `Long` object, will need 20 bytes, and therefore be rounded up to 24 bytes due to alignment. And so we can see that the overhead of using an Object rather than a primitive is: | Object type | Size in bytes | Overhead compared to primitive | -|-------------|---------------|--------------------------------| +| ----------- | ------------- | ------------------------------ | | `Boolean` | 16 | 16x | | `Byte` | 16 | 16x | | `Character` | 16 | 8x | @@ -285,25 +280,25 @@ than a primitive is: | `Double` | 24 | 3x | | `Long` | 24 | 3x | -The above is a "theoretical worst case". If using the recommended factory methods (e.g., the various `valueOf` +The above is a "theoretical worst case". If using the recommended factory methods (e.g., the various `valueOf` overloads), then there is a possibility of leveraging singleton instances coming from a central cache. These cached instances cover both `Boolean` values, all `Byte` values, and up to 256 values (those closest to zero) of all other -non-floating point types. If the cache is used, then effectively the heap size of these instances is zero (i.e., -assuming that they are part of the "base overhead" of the JVM itself, and that they can be amortized across many -references), and therefore their memory cost is only that of the reference itself, which brings us to the following +non-floating point types. If the cache is used, then effectively the heap size of these instances is zero (i.e., +assuming that they are part of the "base overhead" of the JVM itself, and that they can be amortized across many +references), and therefore their memory cost is only that of the reference itself, which brings us to the following point. -An object allocated on the heap is not just floating there on its own, it has at least one pointer referencing it -(otherwise it is unreachable and therefore eligible to garbage collection). That pointer might be on the stack, or it +An object allocated on the heap is not just floating there on its own, it has at least one pointer referencing it +(otherwise it is unreachable and therefore eligible to garbage collection). That pointer might be on the stack, or it might be a field of some other object. Continuing with the example JVM settings above, we would be operating in the -"compressed pointers" mode, meaning that they would take up 4 bytes each (if the max heap size is >= 32 GB, then -pointers cannot be compressed and take up 8 bytes each). And so the size of the pointer needs to be added to the heap +"compressed pointers" mode, meaning that they would take up 4 bytes each (if the max heap size is >= 32 GB, then +pointers cannot be compressed and take up 8 bytes each). And so the size of the pointer needs to be added to the heap size of the object itself, in order to calculate its full memory cost. If we assume the use of recommended factory methods, and we add the pointer size, then we get the following sizes: | Object type | value | Size in bytes | Overhead compared to primitive | -|-------------|-----------|---------------|--------------------------------| +| ----------- | --------- | ------------- | ------------------------------ | | `Boolean` | all | 4 | 4x | | `Byte` | all | 4 | 4x | | `Character` | 0..128 | 4 | 2x | @@ -322,29 +317,33 @@ If we assume the use of recommended factory methods, and we add the pointer size Given the significant overheads described above, it is fair to ask why would we ever want to use the Object version of these types, rather than the primitive version. There are a few reasons: -1. Objects can be null, while primitives cannot. In some cases, the full range of values provided by a primitive type +1. Objects can be null, while primitives cannot. In some cases, the full range of values provided by a primitive type are needed for a given use case, and in addition to that, we also need to model the absence of a value. For example, some use case may require a given field to be not only true, or false, but also absent. This could be modeled by two primitive boolean fields, e.g., having both `boolean configValue` and `boolean configValueIsSet`, though that could - be hard to maintain. Alternatively, we could have a single `Boolean configValue` field, which has the three possible + be hard to maintain. Alternatively, we could have a single `Boolean configValue` field, which has the three possible states we care for (true, false, null). Note that `Optional` is one more way of modeling absence, though in practice - it is not terribly useful, and [we choose to avoid it in the Venice project](how_to/style_guide.md#avoid-optionals). + it is not terribly useful, and + [we choose to avoid it in the Venice project](../development/style-guide.md#avoid-optionals). -2. Objects can be used as a type parameter in a class which has [Generic Types](https://docs.oracle.com/javase/tutorial/java/generics/types.html), - while primitives cannot. For example, a `List` is valid, while a `List` is not. That being said, in the - case of common collections, there is an alternative, provided by one of the dependencies of the Venice project called - [fastutil](https://fastutil.di.unimi.it). In the previous example, `fastutil` could be used to provide an [IntList](https://javadoc.io/doc/it.unimi.dsi/fastutil/latest/it/unimi/dsi/fastutil/ints/IntList.html), - which is an implementation of `List` with additional methods that prevent "boxing" (see below). +2. Objects can be used as a type parameter in a class which has + [Generic Types](https://docs.oracle.com/javase/tutorial/java/generics/types.html), while primitives cannot. For + example, a `List` is valid, while a `List` is not. That being said, in the case of common collections, + there is an alternative, provided by one of the dependencies of the Venice project called + [fastutil](https://fastutil.di.unimi.it). In the previous example, `fastutil` could be used to provide an + [IntList](https://javadoc.io/doc/it.unimi.dsi/fastutil/latest/it/unimi/dsi/fastutil/ints/IntList.html), which is an + implementation of `List` with additional methods that prevent "boxing" (see below). #### Boxing and Unboxing -"Boxing" is the term used to designate Java's support for implicitly converting a primitive type to its Object -equivalent. "Unboxing" is the reverse process, where an Object is transformed into a primitive (though one must be -careful to guard against the fact that unboxing a null Object results a `NullPointerException`). For example, the +"Boxing" is the term used to designate Java's support for implicitly converting a primitive type to its Object +equivalent. "Unboxing" is the reverse process, where an Object is transformed into a primitive (though one must be +careful to guard against the fact that unboxing a null Object results a `NullPointerException`). For example, the following code is valid: ```java class Sample { + Integer boxingExample(int primitiveInt) { Integer objectInteger = primitiveInt; return objectInteger; @@ -352,26 +351,28 @@ class Sample { int unboxingExample(Integer objectInteger) { int primitiveInt = objectInteger; // Warning! This can throw a NullPointerException! - return primitiveInt; + return primitiveInt; } } + ``` -In some cases, the JIT Compiler can optimize away certain object allocations via _scalar replacement_, and boxing can -sometimes be eliminated in this fashion. It should not be assumed, however, that such optimization can take place (nor -that it will even if it can). More details in [JVM Anatomy Quark #18: Scalar Replacement](https://shipilev.net/jvm/anatomy-quarks/18-scalar-replacement/). +In some cases, the JIT Compiler can optimize away certain object allocations via _scalar replacement_, and boxing can +sometimes be eliminated in this fashion. It should not be assumed, however, that such optimization can take place (nor +that it will even if it can). More details in +[JVM Anatomy Quark #18: Scalar Replacement](https://shipilev.net/jvm/anatomy-quarks/18-scalar-replacement/). -In general, therefore, it is recommended to avoid implicit boxing in hot paths where it can be reasonably avoided (i.e., +In general, therefore, it is recommended to avoid implicit boxing in hot paths where it can be reasonably avoided (i.e., without undue maintenance burden). ### Estimating the Heap Size of Objects -The above offers some "rule of thumb" guidance for estimating the heap size of certain kinds of objects, in the hope +The above offers some "rule of thumb" guidance for estimating the heap size of certain kinds of objects, in the hope that it may help developers make informed decisions about class design. But there are also some scenarios within the Venice codebase where we wish to estimate the size on heap of objects at runtime. For example, in the ingestion code, there is some buffering where we wish to deterministically enforce a cap on the size in memory of buffered objects. -There are a few different ways in which we might assess object size in Java, including empirical and theoretical +There are a few different ways in which we might assess object size in Java, including empirical and theoretical approaches. #### Empirical Measurement @@ -379,84 +380,86 @@ approaches. Empirical approaches aim to measure the "true size" of objects. Broadly speaking, we could divide this solution space into two categories, presented below (though each of those could have many variants). -One such way, which we might qualify as a "black box approach", is to ask the JVM how much memory it is using, then -allocating the object of interest, then asking the JVM again how much it's using, and calculating the delta between -before and after. In order to reduce background noise, the allocation step would need to produce a large number of -instances, and hang on to them, so they don't get garbage collected right away. This type of empirical approach is not -very practical within production code (due to inefficiency and occasional imprecision), but we do use it in unit tests +One such way, which we might qualify as a "black box approach", is to ask the JVM how much memory it is using, then +allocating the object of interest, then asking the JVM again how much it's using, and calculating the delta between +before and after. In order to reduce background noise, the allocation step would need to produce a large number of +instances, and hang on to them, so they don't get garbage collected right away. This type of empirical approach is not +very practical within production code (due to inefficiency and occasional imprecision), but we do use it in unit tests to validate how well the other methods are working. -Another way is that some JVM flags or JVM agents can be configured which provide access to measurement functions. This -bucket of experimental approaches are expected to provide full precision. They are also more efficient than the black -box approach (given that there is no need to synthetically allocate a bunch of objects), though it is still not +Another way is that some JVM flags or JVM agents can be configured which provide access to measurement functions. This +bucket of experimental approaches are expected to provide full precision. They are also more efficient than the black +box approach (given that there is no need to synthetically allocate a bunch of objects), though it is still not completely efficient due to needing to rely on [reflection](https://docs.oracle.com/javase/tutorial/reflect/). Moreover, -the requirement to use certain JVM configurations makes these approaches not fully reliable. In Venice's case, given -that measurement was needed in some client library (the Da Vinci Client), where the JVM settings can be quite varied +the requirement to use certain JVM configurations makes these approaches not fully reliable. In Venice's case, given +that measurement was needed in some client library (the Da Vinci Client), where the JVM settings can be quite varied across the user base, this was not considered sufficiently reliable. #### Theoretical Prediction -Rather than empirically measuring, we can use theoretical knowledge of JVM internals to "predict" what the size of a +Rather than empirically measuring, we can use theoretical knowledge of JVM internals to "predict" what the size of a given object _should be_. The downside of this approach is that, given that it is not empirical, it is possible the -theoretical assumptions it is based on are wrong, therefore leading to imprecision. We can further break down this +theoretical assumptions it is based on are wrong, therefore leading to imprecision. We can further break down this bucket into two approaches: -1. **Generic**, where any object of any class can be measured. This kind of approach may be most convenient to - developers but has the downside of requiring reflection, which is not the most efficient. Moreover, achieving - precision requires taking into account whether referenced instances are "shared" (in which case they should not be - counted) or whether, on the contrary, they "belong" to the referencing object (in which case they should be counted). - Keeping track of this phenomenon is costly, both in terms of space and time. For these reasons, the generic - approach was avoided for Venice's measurement use cases. - -2. **Specific**, where only certain classes can be measured. This is the approach chosen in Venice, and it takes the - form of classes implementing the `Measurable` interface. These classes implement a `getHeapSize` function which can - leverage knowledge of the Venice implementation details to skip counting any shared instances, and count - everything else efficiently (i.e., without reflection on the hot path). The downside of course is that this - approach requires more maintenance cost. The utilities in the [com.linkedin.venice.memory](https://venicedb.org/javadoc/com/linkedin/venice/memory/package-summary.html) - package make it easier to implement `Measurable` correctly, though some manual work is still required. +1. **Generic**, where any object of any class can be measured. This kind of approach may be most convenient to + developers but has the downside of requiring reflection, which is not the most efficient. Moreover, achieving + precision requires taking into account whether referenced instances are "shared" (in which case they should not be + counted) or whether, on the contrary, they "belong" to the referencing object (in which case they should be counted). + Keeping track of this phenomenon is costly, both in terms of space and time. For these reasons, the generic approach + was avoided for Venice's measurement use cases. + +2. **Specific**, where only certain classes can be measured. This is the approach chosen in Venice, and it takes the + form of classes implementing the `Measurable` interface. These classes implement a `getHeapSize` function which can + leverage knowledge of the Venice implementation details to skip counting any shared instances, and count everything + else efficiently (i.e., without reflection on the hot path). The downside of course is that this approach requires + more maintenance cost. The utilities in the + [com.linkedin.venice.memory](https://venicedb.org/javadoc/com/linkedin/venice/memory/package-summary.html) package + make it easier to implement `Measurable` correctly, though some manual work is still required. ### Concurrent Access to Memory Java provides first-class support for concurrent workloads. The standard library includes many facilities in the [java.util.concurrent](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/concurrent/package-summary.html) package for writing multithreaded applications. When writing such code, it is often necessary for different threads to -access shared state. In order to do this correctly, it is important to understand the [Java Memory Model](https://docs.oracle.com/javase/specs/jls/se17/html/jls-17.html). +access shared state. In order to do this correctly, it is important to understand the +[Java Memory Model](https://docs.oracle.com/javase/specs/jls/se17/html/jls-17.html). Although a full exploration of this is beyond the scope of this page, a few basic principles can be summarized: -1. Code is not guaranteed to be executed in the order it is written. Due to various optimizations, certain operations +1. Code is not guaranteed to be executed in the order it is written. Due to various optimizations, certain operations which are written sequentially in the code can be re-ordered, or executed in parallel. Occasionally, this can produce surprising results. -2. Memory is not a single entity, but rather many layers of cache, and caches can be incoherent. There is RAM, which - contains the "authoritative" value of a given piece of state, and then there are multiple levels of CPU cache which - copy some subset of the RAM, in order to make it more efficient for the CPU to access it. Threads running on +2. Memory is not a single entity, but rather many layers of cache, and caches can be incoherent. There is RAM, which + contains the "authoritative" value of a given piece of state, and then there are multiple levels of CPU cache which + copy some subset of the RAM, in order to make it more efficient for the CPU to access it. Threads running on different CPUs may have access to different copies of the RAM, and therefore see inconsistent results. 3. In order to achieve correctness when handling state which is read and written by multiple threads, we must give explicit instructions to Java. For example: 1. The `volatile` keyword prepended before a field indicates that reads and writes to that field are guaranteed to be - consistent. Although the way this is guaranteed is not officially specified, in practice it has been - [experimentally determined](https://github.com/alex-dubrouski/java-test?tab=readme-ov-file#volatiles-test) that - volatile writes cause all CPU caches to get invalidated, while the volatile reads can still benefit from CPU - caches. Many of the `java.util.concurrent` classes rely on `volatile` fields to implement various functionalities + consistent. Although the way this is guaranteed is not officially specified, in practice it has been + [experimentally determined](https://github.com/alex-dubrouski/java-test?tab=readme-ov-file#volatiles-test) that + volatile writes cause all CPU caches to get invalidated, while the volatile reads can still benefit from CPU + caches. Many of the `java.util.concurrent` classes rely on `volatile` fields to implement various functionalities such as _Compare and Swap_, locking, etc. - - 2. The `final` keyword prepended before a field indicates that it is immutable, and therefore safe to cache in the - CPU. Note, however, that there is a subtlety. Only a primitive final field is "fully immutable", whereas an Object - final field merely means that the pointer is immutable (i.e., always referencing the same instance on the heap) + + 2. The `final` keyword prepended before a field indicates that it is immutable, and therefore safe to cache in the + CPU. Note, however, that there is a subtlety. Only a primitive final field is "fully immutable", whereas an Object + final field merely means that the pointer is immutable (i.e., always referencing the same instance on the heap) while the fields within the referenced instance can themselves be mutable. - - 3. The `synchronized` keyword can be used to indicate that only one thread at a time is allowed to enter the + + 3. The `synchronized` keyword can be used to indicate that only one thread at a time is allowed to enter the designated critical section. This can make it so that a non-`volatile` field is accessed in a threadsafe way, but only if all such accesses are guarded by synchronization. ## Conclusion -This page is merely a small selection of Java-related topics. Ultimately, learning the language well requires reading +This page is merely a small selection of Java-related topics. Ultimately, learning the language well requires reading and writing a lot of code. -Although most of the above content is general in nature, and only occasionally makes references to Venice-specific -details, a Venice developer should also get familiar with the rest of the project-specific recommendations found in the -[Style Guide](how_to/style_guide.md). \ No newline at end of file +Although most of the above content is general in nature, and only occasionally makes references to Venice-specific +details, a Venice developer should also get familiar with the rest of the project-specific recommendations found in the +[Style Guide](../development/style-guide.md). diff --git a/docs/contributing/architecture/navigation.md b/docs/contributing/architecture/navigation.md new file mode 100644 index 00000000000..7d33062f4f8 --- /dev/null +++ b/docs/contributing/architecture/navigation.md @@ -0,0 +1,57 @@ +# Navigating the Project + +The Venice codebase is split across these directories: + +- `clients`, which contains the user-facing libraries that most Venice users might be interested in. Those include: + - `da-vinci-client`, which is the stateful client, providing "eager caching" for the Venice datasets. + [Learn more](../../user-guide/read-apis/da-vinci-client.md). + - `venice-admin-tool`, which is the shell tool intended for Venice operators. + - `venice-client`, which is a one-stop-shop for all clients which an online application might need, including + thin-client, fast-client, da-vinci-client, consumer. + - `venice-producer`, which enables an application to perform real-time writes to Venice. + - `venice-push-job`, which enables an offline job to push batch data to Venice. + - `venice-thin-client`, which is the most minimal dependency one can get to issue remote reads to the Venice backend, + by delegating as much of the query logic as possible to the Venice router tier. +- `integrations`, which contains additional libraries that some Venice users might be interested in, to connect Venice + with other third-party systems. The rule of thumb for including a module in this directory is that it should have + minimal Venice-specific logic, and be mostly just glue code to satisfy the contracts expected by the third-party + system. Also, these modules are intended to minimize the dependency burden of the other client libraries. Those + include: + - `venice-beam`, which implements the Beam Read API, enabling a Beam job to consume the Venice changelog. + - `venice-pulsar`, which contains an implementation of a Pulsar + [Sink](https://pulsar.apache.org/docs/next/io-overview/#sink), in order to feed data from Pulsar topics to Venice. + - `venice-samza`, which contains an implementation of a Samza + [SystemProducer](https://samza.apache.org/learn/documentation/latest/api/javadocs/org/apache/samza/system/SystemProducer.html), + in order to let Samza stream processing jobs emit writes to Venice. +- `internal`, which contains libraries not intended for public consumption. Those include: + - `alpini`, which is a Netty-based framework used by the router service. It was forked from some code used by + LinkedIn's proprietary + [Espresso](https://engineering.linkedin.com/espresso/introducing-espresso-linkedins-hot-new-distributed-document-store) + document store. At this time, Venice is the only user of this library, so there should be no concern of breaking + compatibility with other dependents. + - `venice-client-common`, which is a minimal set of APIs and utilities which the thin-client and other modules need to + depend on. This module used to be named `venice-schema-common`, as one can see if digging into the git history. + - `venice-common`, which is a larger set of APIs and utilities used by most other modules, except the thin-client. +- `services`, which contains the deployable components of Venice. Those include: + - `venice-controller`, which acts as the control plane for Venice. Dataset creation, deletion and configuration, + schema evolution, dataset to cluster assignment, and other such tasks, are all handled by the controller. + - `venice-router`, which is the stateless tier responsible for routing thin-client queries to the correct server + instance. It can also field certain read-only metadata queries such as cluster discovery and schema retrieval to + take pressure away from the controller. + - `venice-server`, which is the stateful tier responsible for hosting data, serving requests from both routers and + fast-client library users, executing write operations and performing cross-region replication. +- `tests`, which contains some modules exclusively used for testing. Note that unit tests do not belong here, and those + are instead located into each of the other modules above. + +Besides code, the repository also contains: + +- `all-modules`, which is merely an implementation detail of the Venice build and release pipeline. No code is expected + to go here. +- `docker`, which contains our various docker files. +- `docs`, which contains the wiki you are currently reading. +- `gradle`, which contains various hooks and plugins used by our build system. +- `scripts`, which contains a few simple operational scripts that have not yet been folded into the venice-admin-tool. +- `specs`, which contains formal specifications, in TLA+ and FizzBee, for some aspects of the Venice architecture. + +If you have any questions about where some contribution belongs, do not hesitate to reach out on the +[community slack](http://slack.venicedb.org)! diff --git a/docs/dev_guide/router_rest_spec.md b/docs/contributing/architecture/router-api.md similarity index 89% rename from docs/dev_guide/router_rest_spec.md rename to docs/contributing/architecture/router-api.md index 34b9893db8d..701dd46670a 100644 --- a/docs/dev_guide/router_rest_spec.md +++ b/docs/contributing/architecture/router-api.md @@ -1,12 +1,7 @@ ---- -layout: default -title: Router REST Spec -parent: Developer Guides -permalink: /docs/dev_guide/router_rest_spec ---- # Router REST Spec + This page describes how the router's REST endpoints work for reading data out of Venice with the thin client. Currently, -there exists a reference implementation of the thin client in Java, but if clients in additional languages are to be +there exists a reference implementation of the thin client in Java, but if clients in additional languages are to be built, they could reference this spec to do so. Note that although this part of the architecture is RESTful, Venice in general has a bias for performance, so the @@ -17,10 +12,11 @@ The following sections describe the endpoints in the order they are invoked by t order to prime its internal state. As with any client/server interaction, the developer of a new client implementation should consider details such as load -balancing, connection pooling, authentication and authorization. Since these tend to be generic concerns, rather than +balancing, connection pooling, authentication and authorization. Since these tend to be generic concerns, rather than Venice-specific ones, they are omitted from this document. ## Cluster Discovery Endpoint + The first endpoint the client invokes is to discover which cluster the store of interest belongs to: ``` @@ -45,30 +41,33 @@ This will return a JSON response in the following format: For all JSON responses returned by Venice, it is advisable to validate that the `error` field is `null`. If it is not `null`, then it should be a string containing some description of the error. In addition to that, the `errorType` field -is a means to categorize errors programmatically according to the values in the -[ErrorType](https://github.com/linkedin/venice/blob/main/internal/venice-client-common/src/main/java/com/linkedin/venice/exceptions/ErrorType.java) +is a means to categorize errors programmatically according to the values in the +[ErrorType](https://github.com/linkedin/venice/blob/main/internal/venice-client-common/src/main/java/com/linkedin/venice/exceptions/ErrorType.java) enum. The client then uses the information in the `d2Service` field of this response to connect to the correct cluster. A few important details: -1. The cluster discovery mechanism currently makes certain assumptions that - [D2](https://linkedin.github.io/rest.li/Dynamic_Discovery) is used. This assumption could be loosened in the future +1. The cluster discovery mechanism currently makes certain assumptions that + [D2](https://linkedin.github.io/rest.li/Dynamic_Discovery) is used. This assumption could be loosened in the future in order to integrate with other service discovery mechanisms. -2. This step can be skipped if not leveraging multi-cluster deployments, or if the operator wishes to let clients +2. This step can be skipped if not leveraging multi-cluster deployments, or if the operator wishes to let clients hardcode some cluster addresses. -3. If using cluster migration, the mapping between a store and a cluster can change. In that case, there is a way for - the client to find out about it and automatically switch to the new cluster assignment. The switchover process needs +3. If using cluster migration, the mapping between a store and a cluster can change. In that case, there is a way for + the client to find out about it and automatically switch to the new cluster assignment. The switchover process needs to be documented further. ## Schema Endpoints -After connecting to the correct cluster, the client primes its cache of + +After connecting to the correct cluster, the client primes its cache of [Avro](https://avro.apache.org/docs/1.11.1/specification/) schemas, by calling the endpoints below. ### Key Schema Endpoint -The key schema must be used to serialize keys to be queried. In Venice, the key schema cannot evolve, so the client can + +The key schema must be used to serialize keys to be queried. In Venice, the key schema cannot evolve, so the client can rely on this assumption. The key schema is retrieved via: + ``` GET https://:/key_schema/ ``` @@ -87,18 +86,23 @@ This returns a JSON response in the following format: "exceptionType": null } ``` -The significant field here is `schemaStr`, which is the Avro schema in AVSC format, with the double quotes escaped so + +The significant field here is `schemaStr`, which is the Avro schema in AVSC format, with the double quotes escaped so that they do not interfere with the surrounding JSON envelope. ### All Value Schemas Endpoint + In addition to the key schema, a Venice store is also associated with one or many value schemas, which are guaranteed to be fully compatible with one another. A Venice store can contain records encoded with various versions of the schema, so -it is important for the client to know all registered schemas, so that it can perform +it is important for the client to know all registered schemas, so that it can perform [schema evolution](https://avro.apache.org/docs/1.11.1/specification/#schema-resolution). + ``` GET https://:/value_schema/ ``` + This returns the following JSON response: + ```json { "cluster": "cluster_name", @@ -112,7 +116,8 @@ This returns the following JSON response: "derivedSchemaId": -1, "rmdValueSchemaId": -1, "schemaStr": "{\"type\":\"record\",\"name\":\"ValueRecord\",\"namespace\":\"com.foo\",\"fields\":[{\"name\":\"intField\",\"type\":\"int\",\"default\":0}]}" - }, { + }, + { "id": 2, "derivedSchemaId": -1, "rmdValueSchemaId": -1, @@ -122,15 +127,20 @@ This returns the following JSON response: "exceptionType": null } ``` + ### Individual Value Schema Endpoint + After initialization, it is possible that the client encounters new schemas that did not exist yet at the time it was initialized. In those cases, the client can fetch those unknown schemas specifically via: + ``` GET https://:/value_schema// ``` + This returns a payload identical to the key schema endpoint. ## Storage Endpoints + After initialization, the client can begin querying the routers for data. There are three read operations supported in Venice: @@ -147,23 +157,26 @@ X-VENICE-API-VERSION = 1 Furthermore, the following header is optional: ``` -# Indicates that the client is capable of decompressing GZIP responses (if omitted, GZIP compressed stores will be +# Indicates that the client is capable of decompressing GZIP responses (if omitted, GZIP compressed stores will be # decompressed by the router, on behalf of the client). X-VENICE-SUPPORTED-COMPRESSION-STRATEGY = 1 ``` ### Single Get Endpoint + Querying the value for a single key can be achieved by sending an HTTP GET request to: + ``` GET https://:/storage//?f=b64 ``` Alternatively, for stores with a simple string key schema, the value can be gotten via: + ``` GET https://:/storage// ``` -The response code should be 200, and the value is encoded in Avro binary in the body of the response, with the following +The response code should be 200, and the value is encoded in Avro binary in the body of the response, with the following response headers: ``` @@ -175,7 +188,9 @@ X-VENICE-COMPRESSION-STRATEGY = 0 ``` ### Batch Get + Batch gets are performed as an HTTP POST request to: + ``` POST https://:/storage/ ``` @@ -183,7 +198,7 @@ POST https://:/storage/ The request should be accompanied by the following headers: ``` -# Not supplying the streaming header results in a legacy non-streaming mode which is less performant, and which may be +# Not supplying the streaming header results in a legacy non-streaming mode which is less performant, and which may be # removed in the future X-VENICE-STREAMING = 1 @@ -196,7 +211,7 @@ of the keys matter (see details below). The response code will always be 200 since the router starts returning results incrementally before knowing if all parts of the request can be fulfilled successfully (but see the footer details below for the real status code). Furthermore, -the values are laid out one after the other in the body of the response, packaged inside envelopes encoded in the +the values are laid out one after the other in the body of the response, packaged inside envelopes encoded in the following Avro schema: ```json @@ -231,12 +246,12 @@ A few important details: not include the requested keys, but only a `keyIndex` corresponding to the key's position in the request body. 2. The response envelopes are not coming in the same order as they were requested. 3. When a requested key has no associated value, the envelope will contain the following sentinel values: - 1. The `keyIndex` is negative. - 2. The `value` is empty. + 1. The `keyIndex` is negative. + 2. The `value` is empty. 3. The `schemaId` is `-1000`. -4. If any errors occurred as part of handling this request, then a special envelope at the end of the response body is -included, enclosing a "footer record", with the following characteristics: - 1. The `keyIndex` is `-1000000`. +4. If any errors occurred as part of handling this request, then a special envelope at the end of the response body is + included, enclosing a "footer record", with the following characteristics: + 1. The `keyIndex` is `-1000000`. 2. The `value` is encoded using the Avro schema below. 3. The `schemaId` is `-1001`. @@ -271,4 +286,4 @@ included, enclosing a "footer record", with the following characteristics: ### Read Compute -TODO \ No newline at end of file +TODO diff --git a/docs/dev_guide/venice_write_path.md b/docs/contributing/architecture/write-path.md similarity index 75% rename from docs/dev_guide/venice_write_path.md rename to docs/contributing/architecture/write-path.md index 14c961a7dee..3cb661cebed 100644 --- a/docs/dev_guide/venice_write_path.md +++ b/docs/contributing/architecture/write-path.md @@ -1,23 +1,17 @@ ---- -layout: default -title: Venice Write Path -parent: Developer Guides -permalink: /docs/dev_guide/venice_write_path ---- - # Introduction -As a high performance derived data storage platform, Venice is designed to ingest large amount of writes from a variety +As a high performance derived data storage platform, Venice is designed to ingest large amount of writes from a variety of sources while providing low latency read access to the data. These characteristics are achieved by a combination of -design choices and optimizations in the write path. This document describes some internal details of the Venice write path. +design choices and optimizations in the write path. This document describes some internal details of the Venice write +path. -Note that this page is still under construction and only provides glimpses of the Venice writing path. More details will +Note that this page is still under construction and only provides glimpses of the Venice writing path. More details will be added in the future. ### Terminology | Name | Description | -|-------|-------------------------------------------------------------------------------------------------------------------------------| +| ----- | ----------------------------------------------------------------------------------------------------------------------------- | | Store | A store is a table or the dataset in Venice. Each store has multiple partitions. | | VT | Version Topic. The version topic is a kafka topic where holds all the data for a given store. One VT per version per Store. | | RT | Real time Topic. A real time topic is also a kafka topic but it only contains data from nearline producers. One RT per Store. | @@ -29,24 +23,24 @@ be added in the future. ### Venice Store Ingestion on Server Side Helix assign leaders and followers status on each partition to different servers. When assignments are done, each server -will create multiple `StoreIngestionTask`s to handle ingestion for the partitions it's assigned to. Note that there's one SIT per store per -version. Multiple partitions of the same store and version are handled by the same SIT instance. The diagram below -describes the data flow of how a store ingestion task's created and ingest data on the server side. +will create multiple `StoreIngestionTask`s to handle ingestion for the partitions it's assigned to. Note that there's +one SIT per store per version. Multiple partitions of the same store and version are handled by the same SIT instance. +The diagram below describes the data flow of how a store ingestion task's created and ingest data on the server side. -![Server Ingestion Diagram](../assets/images/server_ingestion.svg) +![Server Ingestion Diagram](../../assets/images/server_ingestion.svg) -### Venice Store Ingestion Report +### Venice Store Ingestion Report -When ingestion is in different stages, the server reports the ingestion status. The ingestion signal is propagated to +When ingestion is in different stages, the server reports the ingestion status. The ingestion signal is propagated to various downstream for different business logics. The diagram below describes the signal flow of the ingestion report. -![Server Ingestion Report Diagram](../assets/images/server_ingestion_report.svg) +![Server Ingestion Report Diagram](../../assets/images/server_ingestion_report.svg) ### Quantities When working on the codebase, I find it helpful to keep track of the quantities of different components in the system. -Below, you'll find some visualizations of data structures and other things that are per-store, per-version, and per-partition. -([Link to diagram source](https://whimsical.com/venice-docs-diagrams-SvH4RAc9mED9JdAycS5w2v)) +Below, you'll find some visualizations of data structures and other things that are per-store, per-version, and +per-partition. ([Link to diagram source](https://whimsical.com/venice-docs-diagrams-SvH4RAc9mED9JdAycS5w2v)) ![Write Path Quantities Diagram](https://github.com/user-attachments/assets/cc2dee5d-1513-4f9a-be3b-1a51a53bc7bb) @@ -63,16 +57,16 @@ record must be reconstructed at read-time by using the chunks and manifest in or In the nearline job / partial update path, the `Consumer` must first reconstruct the large record. In `RocksDB`, the original key of the record points to the `ChunkedValueManifest`, which includes a list of keys that point to the chunks. Using these components, the `Consumer` can reconstruct the large record, apply the partial update from RT, and then -divide it into a new set of chunks and a new manifest. The new chunks are written to `RocksDB`, the stored manifest -is updated with the new keys, and the old chunks are deleted. +divide it into a new set of chunks and a new manifest. The new chunks are written to `RocksDB`, the stored manifest is +updated with the new keys, and the old chunks are deleted. -The diagram below illustrates the aforementioned chunking system. ([Link to diagram source](https://whimsical.com/venice-docs-diagrams-SvH4RAc9mED9JdAycS5w2v)) +The diagram below illustrates the aforementioned chunking system. +([Link to diagram source](https://whimsical.com/venice-docs-diagrams-SvH4RAc9mED9JdAycS5w2v)) ![Large Record Value Chunking Diagram](https://github.com/user-attachments/assets/f0ed6eb2-7558-4300-a75c-5fc842bb2fb5) ### Push Job -[More details about push jobs](../user_guide/write_api/push_job) +[More details about push jobs](../../user-guide/write-apis/batch-push.md) ![Venice Push Job Diagram](https://github.com/user-attachments/assets/bdcefd28-a550-4ab8-9e84-5654efdcb0c0) - diff --git a/docs/contributing/code-of-conduct.md b/docs/contributing/code-of-conduct.md new file mode 100644 index 00000000000..b002d4fd8e9 --- /dev/null +++ b/docs/contributing/code-of-conduct.md @@ -0,0 +1,106 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for +everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity +and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, +color, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take +appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, +issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for +moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing +the community in public spaces. Examples of representing our community include using an official e-mail address, posting +via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible +for enforcement over [Slack](https://venicedb.slack.com/archives/C03SLQWRSLF). All complaints will be reviewed and +investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem +in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the +community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation +and an explanation of why the behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including +unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding +interactions in community spaces as well as external channels like social media. Violating these terms may lead to a +temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified +period of time. No public or private interaction with the people involved, including unsolicited interaction with those +enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate +behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md new file mode 100644 index 00000000000..414e0a0e288 --- /dev/null +++ b/docs/contributing/contributing.md @@ -0,0 +1,62 @@ +# Contribution Agreement + +As a contributor, you represent that the code you submit is your original work or that of your employer (in which case +you represent you have the right to bind your employer). By submitting code, you (and, if applicable, your employer) are +licensing the submitted code to LinkedIn and the open source community subject to the BSD 2-Clause license. + +# Responsible Disclosure of Security Vulnerabilities + +Please refer to our [Security Policy](security.md) for our policy on responsibly reporting security vulnerabilities. + +# Contribution Process + +The Venice community welcome everyone, and encourage a friendly and positive environment. + +**Contributions of various forms are welcome!** + +Please read existing GitHub issues or development work that is in progress or in the backlog to avoid duplication. If +you are interested in those existing ones, you can leave a comment in the GitHub issues and the community will try to +involve you. If you are not sure if it's duplicated, just create a GitHub issue and ask! + +For any PR, a GitHub issue is required. + +If you want to contribute something new, and it's not tracked in existing GitHub issues, please create a new GitHub +issue and the community will help review the idea. Please state `why` in your GitHub issue. If you already have a short +design in mind or change is not small please check the [Venice Improvement Proposal](documentation/design-docs.md) + +If you have any feature request, please create a new GitHub issue and the community will collect your feature request +and work on it. + +If you have any user feedback, please create a new GitHub issue and the community will collect your feedback and work on +it. + +## For New Contributors + +Please follow [Venice Workspace Setup](development/workspace-setup.md) and +[Venice Recommended Development Workflow](development/dev-workflow.md) + +## Tips for Getting Your Pull Request Accepted + +1. Make sure all new features are tested and the tests pass. +2. Bug fixes must include a test case demonstrating the error that it fixes. +3. Open an issue first and seek advice for your change before submitting a pull request. Large features which have never + been discussed are unlikely to be accepted. + +## Community Syncup Meeting + +Venice holds virtual meetings for the community to come together to syncup. + +1. Meeting Frequency: Bi-weekly on Monday at 9 AM Pacific, 12 PM Eastern, 6 PM Central Europe +2. Meeting Co-ordination: Please join the #community-syncup channel in the + [public Venice Slack](http://slack.venicedb.org/) +3. Meeting Length: One hour +4. Agenda: + 1. Discuss Venice Improvement Proposals. + 2. Review and address concerns for any open pull requests. + 3. Open discussion. + +## Code of Conduct + +This project and everyone who participates in it is governed by the [Venice Code of Conduct](code-of-conduct.md). By +participating, you are expected to uphold this code. Please report unacceptable behavior on +[Slack](https://venicedb.slack.com/archives/C03SLQWRSLF). diff --git a/docs/contributing/development/dev-workflow.md b/docs/contributing/development/dev-workflow.md new file mode 100644 index 00000000000..08a5e9b6aeb --- /dev/null +++ b/docs/contributing/development/dev-workflow.md @@ -0,0 +1,118 @@ +# Venice Recommended Development Workflow + +## Create a Design Document + +If your change is relatively minor, you can skip this step. If you are adding new major functionality, we suggest that +you add a design document and solicit comments from the community before submitting any code. + +Please follow the [Design Document Guide](../documentation/design-docs.md). + +## Creating GitHub issue + +Every PR should be preceded by a GitHub issue to explain the problem statement unless it's a trivial bug fixes or a +documentation change. If your change is significant, please make sure your PR reviewers can align on the problem +statement via GitHub issues first. + +The GitHub issue should contain the detailed problem statement. + +## Pull Request + +1. Fork the GitHub repository at [http://github.com/linkedin/venice](http://github.com/linkedin/venice) if you haven't + already +2. Clone your fork, create a new branch, push commits to the branch +3. Consider whether documentation or tests need to be added or updated as part of the change, and add them as needed + (doc changes should be submitted along with code change in the same PR) +4. Run all tests as described in the project's [Workspace setup guide](workspace-setup.md#run-the-test-suite). +5. Open a pull request against the `main` branch of `linkedin/venice`. (Only in special cases would the PR be opened + against other branches.) +6. The PR title should usually be of the form `[component1]...[componentN]: Concise commit message`. + - Valid tags are: `[da-vinci]` (or `[dvc]`), `[server]`, `[controller]`, `[router]`, `[samza]`, `[vpj]`, + `[fast-client]` (or `[fc]`), `[thin-client]` (or `[tc]`), `[changelog]` (or `[cc]`), `[producer]`, `[admin-tool]`, + `[test]`, `[build]`, `[doc]`, `[script]`, `[compat]`, `[protocol]` + - `[compat]` tag means there are compatibility related changes in this PR, including upgrading protocol version, + upgrading system store value schemas, etc. When there is a compatibility related change, it usually requires a + specific deployment order, like upgrading controller before upgrading server. In this case, please explicitly call + out the required deployment order in the commit message. +7. If the pull request is still a work in progress, and so is not ready to be merged, but needs to be pushed to GitHub + to facilitate review, then create the PR as a + [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) + PR + - If the PR cannot be created as a + [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) + PR, add `[WIP]` before the list of components. +8. Please state that the contribution is your original work and that you license the work to the project under the + project's open source license. +9. If this PR resolves an issue be sure to include `Resolves #XXX` to correctly link and close the issue upon merge. +10. The project uses Apache Jenkins for continuous testing on Linux AMD64 and ARM64 build nodes. A CI job will not be + started automatically for pull request. A maintainer has to trigger the testing. Feel free to tag a maintainer and + ask for a build trigger. +11. Once ready, a maintainer will update the PR with the test results. +12. Investigate and fix failures caused by the pull the request +13. Fixes can simply be pushed to the same branch from which you opened your pull request. +14. Please address feedback via additional commits instead of amending existing commits. This makes it easier for the + reviewers to know what has changed since the last review. All commits will be squashed into a single one by the + committer via GitHub's squash button or by a script as part of the merge process. +15. Jenkins will automatically re-test when new commits are pushed. +16. Despite our efforts, Venice may have flaky tests at any given point, which may cause a build to fail. You need to + ping committers to trigger a new build. If the failure is unrelated to your pull request and you have been able to + run the tests locally successfully, please mention it in the pull request. + +## PR Description + +Describe + +- What changes to make and why you are making these changes. +- How are you going to achieve your goal +- Describe what testings you have done, for example, performance testing etc. + +Checklist that might be helpful to facilitate the review: + +- Design one-pager, design doc, or RFC +- GitHub Issue + +### Added new dependencies? + +Please list the new dependencies in the PR description and answer these questions for each new dependency + +- What's their license? +- Are they compatible with our license? + +## The Review Process + +1. Other reviewers, including committers, may comment on the changes and suggest modifications. Changes can be added by + simply pushing more commits to the same branch. +2. Please add a comment and "@" the reviewer in the PR if you have addressed reviewers' comments. Even though GitHub + sends notifications when new commits are pushed, it is helpful to know that the PR is ready for review once again. +3. Lively, polite, rapid technical debate is encouraged from everyone in the community. The outcome may be a rejection + of the entire change. +4. Reviewers can indicate that a change looks suitable for merging by approving it via GitHub's review interface. This + indicates the strongest level of technical sign-off on a patch and it means: "I've looked at this thoroughly and take + as much ownership as if I wrote the patch myself". If you approve a pull request, you will be expected to help with + bugs or follow-up issues on the patch. Consistent, judicious use of pull request approvals is a great way to gain + credibility as a reviewer with the broader community. Venice reviewers will typically include the acronym LGTM in + their approval comment. This was the convention used to approve pull requests before the "approve" functionality was + introduced by GitHub. +5. Sometimes, other changes will be merged which conflict with your pull request's changes. The PR can't be merged until + the conflict is resolved. This can be resolved with `"git fetch upstream"` followed by `"git rebase upstream/main"` + and resolving the conflicts by hand, then pushing the result to your branch. +6. Try to be responsive to the discussion rather than let days pass between replies. + +## Closing Your Pull Request / Issue + +1. If a change is accepted, it will be merged and the pull request will automatically be closed, along with the + associated Issue if the PR description contains `Resolves #XXX` +2. Note that in the rare case you are asked to open a pull request against a branch besides `main`, that you will + actually have to close the pull request manually +3. If your pull request is ultimately rejected, please close it. +4. If a pull request has gotten little or no attention, consider improving the description or the change itself and ping + likely reviewers after a few days. Consider proposing a change that's easier to include, like a smaller and/or less + invasive change. +5. If a pull request is closed because it is deemed not the right approach to resolve an Issue, then leave the Issue + open. However, if the review makes it clear that the problem identified in the Issue is not going to be resolved by + any pull request (not a problem, won't fix) then also resolve the Issue. + +## Attribution & Acknowledgements + +This guide is based on the +[Contributing Code Changes](https://cwiki.apache.org/confluence/display/KAFKA/Contributing+Code+Changes) guide from the +[Apache Kafka](https://kafka.apache.org/) project. diff --git a/docs/dev_guide/how_to/style_guide.md b/docs/contributing/development/style-guide.md similarity index 76% rename from docs/dev_guide/how_to/style_guide.md rename to docs/contributing/development/style-guide.md index 9d257e913e4..51f41cc73da 100644 --- a/docs/dev_guide/how_to/style_guide.md +++ b/docs/contributing/development/style-guide.md @@ -1,45 +1,32 @@ ---- -layout: default -title: Style Guide -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/style_guide ---- - # Style Guide -{: .no_toc } -This page describes some stylistic concerns which the development team cares about. Some of them are enforced by -automation, while others are guidelines of a more informal, or even philosophical, nature. More generally, we believe in -acquiring a deep understanding of the principles behind these guidelines, and being thoughtful about which situation +This page describes some stylistic concerns which the development team cares about. Some of them are enforced by +automation, while others are guidelines of a more informal, or even philosophical, nature. More generally, we believe in +acquiring a deep understanding of the principles behind these guidelines, and being thoughtful about which situation they apply or don't apply to. We don't buy into mainstream ideas such as "all coupling is bad", "all optimizations are premature", etc. We take this common wisdom and incorporate it into our reflections, without blindly taking it at face value. -We also want to acknowledge that our conclusions change over time, either due to hindsight or because of the evolving -context that the project needs to navigate. As such, this should be thought of as a living document, and it is therefore -natural that not all parts of the code base adhere to it perfectly. When such deviations are discovered, it is -encouraged to try to rectify them "in passing", even if the objective of the code change is unrelated. This, too, is a +We also want to acknowledge that our conclusions change over time, either due to hindsight or because of the evolving +context that the project needs to navigate. As such, this should be thought of as a living document, and it is therefore +natural that not all parts of the code base adhere to it perfectly. When such deviations are discovered, it is +encouraged to try to rectify them "in passing", even if the objective of the code change is unrelated. This, too, is a judgment call, and needs to be balanced by concerns like managing the risk of unintended regressions. -For those who are new to the Java language, it may be useful to read this [introduction to Java](../java.md). - -## Table of Contents -{: .no_toc } - -- TOC -{:toc} +For those who are new to the Java language, it may be useful to read this +[introduction to Java](../architecture/java-internals.md). ## Automation Regarding code style, we use the Eclipse Java Formatter variant of Spotless, which automatically reformats the code as part of git commit hooks. Make sure to run `./gradlew assemble` to get the git hooks set up. -We also use Spotbugs, with [some of the rules](https://github.com/linkedin/venice/blob/main/gradle/spotbugs/include.xml) -resulting in build failures if violated. Over time, we intend to pick up more of these rules, fix the code to comply -with them, and add them to the list. If you would like to contribute to this effort, feel free to open an issue and -suggest which rules you are interested in fixing. Note that there are [a few rules that we intend to ignore](https://github.com/linkedin/venice/blob/main/gradle/spotbugs/exclude.xml) -as they seem to be imprecise or not sufficiently useful. +We also use Spotbugs, with [some of the rules](https://github.com/linkedin/venice/blob/main/gradle/spotbugs/include.xml) +resulting in build failures if violated. Over time, we intend to pick up more of these rules, fix the code to comply +with them, and add them to the list. If you would like to contribute to this effort, feel free to open an issue and +suggest which rules you are interested in fixing. Note that there are +[a few rules that we intend to ignore](https://github.com/linkedin/venice/blob/main/gradle/spotbugs/exclude.xml) as they +seem to be imprecise or not sufficiently useful. In the future, we might add other forms of automation to increase code quality along other dimensions. Feel free to suggest ideas in this space. @@ -48,22 +35,22 @@ suggest ideas in this space. Below are a set of guidelines to take into consideration when developing in the Venice project. They are not absolute rules and there may be good reasons to deviate from them occasionally. When deviating, it is useful to leave comments -explaining why we deviated, whether it was intentional, or due to the need for expediency. This helps future maintainers +explaining why we deviated, whether it was intentional, or due to the need for expediency. This helps future maintainers understand what is actually worth cleaning up and how careful they need to be when doing it. ### Compatibility We care about compatibility across versions of the software. This is a bidirectional statement. Old clients need to be able to talk with new servers, and new clients with old servers as well. It also includes interactions across lifetimes -of the same process, for example, state persisted by an old version of the server code should be usable by a newer +of the same process, for example, state persisted by an old version of the server code should be usable by a newer version of the server code, and vice versa. If compatibility is impossible, then we should look for ways to achieve correct behavior anyway (e.g. potentially at the cost of efficiency, such as the server needing to throw away and regenerate the state). -For enums which are going to be communicated across processes or across lifetimes of the same process, consider using -[VeniceEnumValue](http://venicedb.org/javadoc/com/linkedin/venice/utils/VeniceEnumValue.html), [EnumUtils](http://venicedb.org/javadoc/com/linkedin/venice/utils/EnumUtils.html) -and related unit test classes, which provide a structure to minimize the chance that we mistakenly change the mapping of -numeric ID -> enum value. +For enums which are going to be communicated across processes or across lifetimes of the same process, consider using +[VeniceEnumValue](http://venicedb.org/javadoc/com/linkedin/venice/utils/VeniceEnumValue.html), +[EnumUtils](http://venicedb.org/javadoc/com/linkedin/venice/utils/EnumUtils.html) and related unit test classes, which +provide a structure to minimize the chance that we mistakenly change the mapping of numeric ID -> enum value. ### JavaDoc @@ -73,7 +60,7 @@ an indicator that the class ought to be split up. JavaDoc for functions is desired for public client APIs intended to be leveraged directly by end users. For internal functions, JavaDoc is desired only if there is something important to call out. Sometimes, a lengthy function JavaDoc -may be an indication that the function's name is not sufficiently clear, or that the complex function should be split +may be an indication that the function's name is not sufficiently clear, or that the complex function should be split into multiple simpler (and well-named!) functions. Use a single-line JavaDoc if it fits, e.g., `/** A very short description */` @@ -85,21 +72,21 @@ be configured automatically when calling: `./gradlew cleanIdea idea` ### Logging -We use log4j2 and want to use interpolation, rather than manual string concatenation, everywhere for efficiency reasons. +We use log4j2 and want to use interpolation, rather than manual string concatenation, everywhere for efficiency reasons. Note that Spotless may break up fragments of strings by concatenating over multiple lines, but that doesn't matter as it gets optimized away by the compiler. Only concatenations with variables end up carrying an overhead. -Hot path logging should not be above debug level. Keep in mind that exception paths could occasionally turn into hot +Hot path logging should not be above debug level. Keep in mind that exception paths could occasionally turn into hot paths. In those cases, we may want to use the `RedundantExceptionFilter`. More generally, hot path logging typically benefits from being converted into a metric instead. Do your best to make log messages meaningful. Avoid scary yet imprecise wording. If the situation is dire, let the log spell out precisely what happened, along with enough context to make debugging easier. -If there is a known solution to remediate the issue, consider why isn't this solution reactively activated so the system -fixes itself, rather than just logging? If the solution exists but cannot be wired in reactively, then it may be -desirable for the log to indicate what that solution is, to give the user or operator a clue about what to do next -(i.e. the _Ghost in the Shell_ design pattern). +If there is a known solution to remediate the issue, consider why isn't this solution reactively activated so the system +fixes itself, rather than just logging? If the solution exists but cannot be wired in reactively, then it may be +desirable for the log to indicate what that solution is, to give the user or operator a clue about what to do next (i.e. +the _Ghost in the Shell_ design pattern). ### Encapsulation @@ -119,15 +106,15 @@ perhaps the multiple functions should be presented as a single public function, private functions, in the correct order. Avoid passing `this` into classes as this may make the flow of the code difficult to understand. Also, more generally, -consider whether a class actually needs a handle of an instance of an entire other class (and thus have the ability to -call any of its functions), or whether it could make do with an instance of a more constrained interface, which the -other class implements, or perhaps even just a handle to a specific function of the other class (thus limiting the +consider whether a class actually needs a handle of an instance of an entire other class (and thus have the ability to +call any of its functions), or whether it could make do with an instance of a more constrained interface, which the +other class implements, or perhaps even just a handle to a specific function of the other class (thus limiting the surface area of their interaction). ### Avoid Wildcard Imports -We avoid wildcard imports since they may lead to bugs due to pulling in unintended classes or functions. If using -IntelliJ, the auto-conversion of imports into wildcards can be disabled by following these +We avoid wildcard imports since they may lead to bugs due to pulling in unintended classes or functions. If using +IntelliJ, the auto-conversion of imports into wildcards can be disabled by following these [instructions](https://www.jetbrains.com/help/idea/creating-and-optimizing-imports.html#disable-wildcard-imports). This would be a good candidate for automation, perhaps via a new Spotbugs plugin; contributions welcome! @@ -135,9 +122,9 @@ would be a good candidate for automation, perhaps via a new Spotbugs plugin; con We are aligned with the philosophy of the original creators of the Optional API, which is that it is a useful construct in the context of the Java 8 stream APIs, but should generally not be used beyond that. Null is a perfectly appropriate -way to denote emptiness, and is not more or less likely to cause a `NullPointerException`. Sentinel values in primitive +way to denote emptiness, and is not more or less likely to cause a `NullPointerException`. Sentinel values in primitive types (e.g., `-1` for a numeric value that is otherwise expected to be positive) are also perfectly appropriate ways to -denote emptiness. For more info, here are a good [video](https://www.youtube.com/watch?v=fBYhtvY19xA&t=2317s) and +denote emptiness. For more info, here are a good [video](https://www.youtube.com/watch?v=fBYhtvY19xA&t=2317s) and [post](https://homes.cs.washington.edu/~mernst/advice/nothing-is-better-than-optional.html) on this subject. ### Avoid Java Stream API in Hot Paths @@ -157,6 +144,7 @@ interface MyInterface { } class Sample { + void sample() { MyInterface anonymousClassInstance = new MyInterface() { @Override @@ -166,6 +154,7 @@ class Sample { }; } } + ``` Java also supports static blocks, which are executed once the first time a class is loaded. For example: @@ -175,11 +164,12 @@ class Sample2 { static { System.out.println("This will always be printed first, and only once."); } - + void sample() { System.out.println("This will be printed everytime the function is called, but always after the static block."); } } + ``` There is an antipattern which consists of initializing a collection (such as a map) with "double braces" in order to add @@ -188,6 +178,7 @@ anonymous class, and the second (inner) set of braces denoting a static block. F ```java class Sample3 { + void sample() { // antipattern: Map map1 = new HashMap<>() { @@ -201,69 +192,67 @@ class Sample3 { Map map2 = new HashMap<>(); map2.put("k1", "v1"); map2.put("k2", "v2"); - + // other correct way: - Map map3 = CollectionUtil.mapBuilder() - .put("k1", "v1") - .put("k2", "v2") - .build(); + Map map3 = CollectionUtil.mapBuilder().put("k1", "v1").put("k2", "v2").build(); } } + ``` -The reason to avoid the above antipattern is that anonymous classes take up memory in the JVM's metaspace, and we -only wish to pay this overhead for cases where we really need a separate class. The double brace style is not the only -way of populating a map, nor is it even the least verbose way to do it, so there is no point in doing it this way. +The reason to avoid the above antipattern is that anonymous classes take up memory in the JVM's metaspace, and we only +wish to pay this overhead for cases where we really need a separate class. The double brace style is not the only way of +populating a map, nor is it even the least verbose way to do it, so there is no point in doing it this way. ### Look for Ways to Mitigate Failures In a system with lots of moving parts, it should be expected that things fail all the time. We should look for design patterns that help us mitigate failures, wherever possible. -An example of this is the way that dataset versions work in Venice. A dataset version has a unique name, based on the +An example of this is the way that dataset versions work in Venice. A dataset version has a unique name, based on the dataset name concatenated with a monotonically increasing version number. A given dataset version name is immutable, in the sense that it will forever point to one and only dataset version, and cannot be reused. Even if a dataset is deleted -and then re-created under the same name, we don't restart the version number sequence, so there cannot be a clash in -the names of dataset versions coming from before and after the dataset re-creation. A dataset version is associated with +and then re-created under the same name, we don't restart the version number sequence, so there cannot be a clash in the +names of dataset versions coming from before and after the dataset re-creation. A dataset version is associated with various resources including a Kafka topic, a Helix resource, persistent state, and in-memory state. When purging an old dataset version, if any of the resources that constitute it fail to get cleaned up properly, it doesn't prevent future dataset versions from continuing to get created, since they should never clash. In this case, therefore, a failure to -delete a resource results not in immediate and widespread systemic failure, but merely in a resource leak, which can be +delete a resource results not in immediate and widespread systemic failure, but merely in a resource leak, which can be monitored, alerted on, and remediated if it creeps beyond a certain threshold. ### Be a Benevolent Tyrant The CPU will do whatever we tell it, day in day out, without complaints, but it does not mean we ought to abuse it. -Although there is undoubtedly a kernel of truth in the saying that "premature optimization is the root of all evil", -it is important to consider that the reverse is not equally true. In other words, non-optimized code is not the root of -all clean code. This picture from one of the talks by Java performance expert Aleksey Shipilëv describes the idea in an -easy to grasp manner: +Although there is undoubtedly a kernel of truth in the saying that "premature optimization is the root of all evil", it +is important to consider that the reverse is not equally true. In other words, non-optimized code is not the root of all +clean code. This picture from one of the talks by Java performance expert Aleksey Shipilëv describes the idea in an easy +to grasp manner: ![Complexity vs Performance](https://user-images.githubusercontent.com/1248632/195111861-518f81c4-f226-4942-b88a-a34337da79e3.png) -For example, if a class contains some final string property, and the code in this class repeatedly performs a lookup -by that property, then it implies that the result of this lookup may change over time. If that is true, then the code is +For example, if a class contains some final string property, and the code in this class repeatedly performs a lookup by +that property, then it implies that the result of this lookup may change over time. If that is true, then the code is fine, but if it is not true that the result of the lookup would change over time, then it is simply useless code. Doing -the lookup just once, and caching the result in another final property, makes the code not only faster and more +the lookup just once, and caching the result in another final property, makes the code not only faster and more efficient, but also easier to reason about, since it indicates the immutability of this looked up property. Another example is interrogating a map to see if it contains a key, and if true, then getting that key out of the map. This requires 2 lookups, whereas in fact only 1 lookup would suffice, as we can get a value from the map and then check whether it's null. Moreover, doing it in 1 lookup is actually cleaner, since it eliminates the race condition where the -lookup may exist during the `containsKey` check but then be removed prior to the subsequent `get` call. Again, the +lookup may exist during the `containsKey` check but then be removed prior to the subsequent `get` call. Again, the faster code is cleaner. Yet another example is using the optimal data structure for a given use case. A frequent use case within Venice is to look something up by partition number (which are in a tight range, between zero and some low number), and thus it is -possible do the job with either an int-keyed map or an array. If both work equally well from a functional standpoint, -then let us use an array, as it is more efficient to perform an index lookup within an array than a map lookup. For -collections of primitives, it is advised to consider using [fastutil](https://fastutil.di.unimi.it/). If using a more -efficient data structure requires significant acrobatics, then we may still prefer to opt for the less efficient one, -for the sake of maintainability (e.g., if it falls within the red zone of the above diagram). That being said, we should +possible do the job with either an int-keyed map or an array. If both work equally well from a functional standpoint, +then let us use an array, as it is more efficient to perform an index lookup within an array than a map lookup. For +collections of primitives, it is advised to consider using [fastutil](https://fastutil.di.unimi.it/). If using a more +efficient data structure requires significant acrobatics, then we may still prefer to opt for the less efficient one, +for the sake of maintainability (e.g., if it falls within the red zone of the above diagram). That being said, we should consider whether we can build a new data structure which achieves both convenience and efficiency for a given use case -(e.g., yellow zone). This kind of low-level work is not considered off-limits within Venice, and we welcome it if there +(e.g., yellow zone). This kind of low-level work is not considered off-limits within Venice, and we welcome it if there is a good rationale for it. -More generally, always keep in mind that the hot path in Venice may be invoked hundreds of thousands of times per second -per server, and it is therefore important to minimize overhead in these paths. By being benevolent tyrants, our CPUs -serve us better, and will hopefully care for us when AGI takes over the world. \ No newline at end of file +More generally, always keep in mind that the hot path in Venice may be invoked hundreds of thousands of times per second +per server, and it is therefore important to minimize overhead in these paths. By being benevolent tyrants, our CPUs +serve us better, and will hopefully care for us when AGI takes over the world. diff --git a/docs/dev_guide/how_to/code_coverage_guide.md b/docs/contributing/development/testing.md similarity index 53% rename from docs/dev_guide/how_to/code_coverage_guide.md rename to docs/contributing/development/testing.md index 6f96756e424..9ccd7ce3a97 100644 --- a/docs/dev_guide/how_to/code_coverage_guide.md +++ b/docs/contributing/development/testing.md @@ -1,27 +1,22 @@ ---- -layout: default -title: Code Coverage Guide -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/code_coverage_guide ---- # Code Coverage Guide -The Venice repository has two code coverage check on GitHub Action, submodule-level coverage verification -and new-commit coverage verification, which aim to improve the testability and code quality overall. The -GitHub Action fails the Pull Request if the coverage check doesn't meet the requirements. This guide provides the details -on how these reports are generated and how to debug them. +The Venice repository has two code coverage check on GitHub Action, submodule-level coverage verification and new-commit +coverage verification, which aim to improve the testability and code quality overall. The GitHub Action fails the Pull +Request if the coverage check doesn't meet the requirements. This guide provides the details on how these reports are +generated and how to debug them. ## Module structure -Venice has 4 modules, `all-modules`, `clients`, `internal` and `services`. Each module has its own submodules, except `all-modules`. +Venice has 4 modules, `all-modules`, `clients`, `internal` and `services`. Each module has its own submodules, except +`all-modules`. #### Clients `da-vinci-client`, `venice-admin-tool`, `venice-client`, `venice-push-job`, `venice-samza`, `venice-thin-client` #### Internal -`alpini`, `venice-avro-compatibility-test`, `venice-client-common`, `venice-common`, `venice-consumer`, + +`alpini`, `venice-avro-compatibility-test`, `venice-client-common`, `venice-common`, `venice-consumer`, `venice-jdk-compatibility-test`, `venice-test-common` #### Services @@ -31,39 +26,41 @@ Venice has 4 modules, `all-modules`, `clients`, `internal` and `services`. Each ## Submodule-level coverage verification ### Description -This will check the overall code coverage at the submodule level. -Jacoco generates the report based off the current submodule, performs the coverage verification and fails the build if the -coverage is below the threshold. The threshold, targets at the branch coverage, is defined in the `build.gradle` of each -submodule independently. + +This will check the overall code coverage at the submodule level. Jacoco generates the report based off the current +submodule, performs the coverage verification and fails the build if the coverage is below the threshold. The threshold, +targets at the branch coverage, is defined in the `build.gradle` of each submodule independently. ### Example commands + ```Bash # Template -./gradlew :::jacocoTestCoverageVerification +./gradlew :::jacocoTestCoverageVerification -./gradlew :clients:venice-push-job:jacocoTestCoverageVerification +./gradlew :clients:venice-push-job:jacocoTestCoverageVerification ``` The report will be located at `//build/reports/jacoco/test/index.html`. - ### Debugging notes #### Run commands against module doesn't generate the report -Due to the current project setup, running the commands, e.g. `./gradlew :clients:jacocoTestCoverageVerification`, doesn't -execute the unit tests thus no jacoco report will be generated. Please be sure to run the commands against the submodule. +Due to the current project setup, running the commands, e.g. `./gradlew :clients:jacocoTestCoverageVerification`, +doesn't execute the unit tests thus no jacoco report will be generated. Please be sure to run the commands against the +submodule. ## New-commit coverage verification ### Description -This will check the overall code coverage at the commit-level. -DiffCoverage, which is an extension of Jacoco, gathers the diff files by comparing the local branch and remote upstream -main branch, and leverages the Jacoco report, to re-generate a new report only for these newly added lines/files. Similarly, -it performs the coverage verification and fails the build if the coverage is below the threshold. The threshold is defined -at 60% for branch coverage. + +This will check the overall code coverage at the commit-level. DiffCoverage, which is an extension of Jacoco, gathers +the diff files by comparing the local branch and remote upstream main branch, and leverages the Jacoco report, to +re-generate a new report only for these newly added lines/files. Similarly, it performs the coverage verification and +fails the build if the coverage is below the threshold. The threshold is defined at 60% for branch coverage. ### Example commands + ```Bash # Template ./gradlew :::jacocoTestCoverageVerification diffCoverage --continue @@ -76,21 +73,25 @@ The report will be located at `/build/reports/jacoco/diffCoverage/h ### Debugging notes #### Integration tests are added but DiffCoverage doesn't identify the code coverage of it -Though integration tests are strongly encouraged, both Jacoco and DiffCoverage only work with unit tests so please consider writing -unit tests. + +Though integration tests are strongly encouraged, both Jacoco and DiffCoverage only work with unit tests so please +consider writing unit tests. #### Unit tests are added but the DiffCoverage report doesn't reflect my changes There are two possible reasons. -1. **Jacoco report isn't up-to-date**. DiffCoverage report relies on the Jacoco Report to reflect the correct coverage. -If the Jacoco report isn't up-to-date, for example, executing the wrong command `./gradlew :clients:venice-push-job:diffCoverage` which -misses the step of re-running unit tests and updating the Jacoco report, can cause this issue. + +1. **Jacoco report isn't up-to-date**. DiffCoverage report relies on the Jacoco Report to reflect the correct coverage. + If the Jacoco report isn't up-to-date, for example, executing the wrong command + `./gradlew :clients:venice-push-job:diffCoverage` which misses the step of re-running unit tests and updating the + Jacoco report, can cause this issue. 1. Please be sure to run test and Jacoco report first. -2. **Units tests are placed in a different module**. Jacoco can only analyze and generate reports based off the current -submodule. So, if you write new source codes in `submodule A` and `submodule B`, and you only have written unit tests in -`submodule B` which cover changes in `submodule A`, this cannot be recognized by Jacoco thus the DiffCoverage doesn't think -there's coverage too. - 1. Please move/re-organize some unit tests to the right submodule such that the coverage can be detected and reported. +2. **Units tests are placed in a different module**. Jacoco can only analyze and generate reports based off the current + submodule. So, if you write new source codes in `submodule A` and `submodule B`, and you only have written unit tests + in `submodule B` which cover changes in `submodule A`, this cannot be recognized by Jacoco thus the DiffCoverage + doesn't think there's coverage too. + 1. Please move/re-organize some unit tests to the right submodule such that the coverage can be detected and + reported. #### The DiffCoverage report shows some files don't belong to my local changes @@ -98,8 +99,8 @@ That's usually due to your local branch and upstream is not up-to-date so when i `linkedin/venice` are mistakenly treated as your changes. Please do the followings: + 1. Go to your Github fork(https://github.com//venice) and sync the `main` branch with upstream `linkedin:main` 2. Run `git fetch upstream` locally and pull the latest changes to your `main` branch 3. Merge `main` branch to your feature branch. 4. Confirm the diff are only your changes by running `git diff upstream/main`. - diff --git a/docs/dev_guide/how_to/workspace_setup.md b/docs/contributing/development/workspace-setup.md similarity index 75% rename from docs/dev_guide/how_to/workspace_setup.md rename to docs/contributing/development/workspace-setup.md index c9c6ed4659d..28809af266e 100644 --- a/docs/dev_guide/how_to/workspace_setup.md +++ b/docs/contributing/development/workspace-setup.md @@ -1,19 +1,14 @@ ---- -layout: default -title: Venice Workspace Setup -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/workspace_setup ---- - # Venice Workspace Setup -We recommend using a Unix-based environment for development, such as Linux or macOS. -If you're on Windows, we recommend using [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install). + +We recommend using a Unix-based environment for development, such as Linux or macOS. If you're on Windows, we recommend +using [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install). ## Fork the Venice Repository + Fork the Venice repo at [https://github.com/linkedin/venice](https://github.com/linkedin/venice). ## Setting up the repository locally + ```shell git clone git@github.com:${githubUsername}/venice.git cd venice @@ -22,24 +17,26 @@ git fetch upstream ``` ## Setting up Java -We use Java 17 for development. You can download it [here](https://www.oracle.com/java/technologies/javase/jdk17-archive-downloads.html). -Make sure to set the `JAVA_HOME` environment variable to the location of your JDK installation. -How to do this will be dependent on your OS. +We use Java 17 for development. You can download it +[here](https://www.oracle.com/java/technologies/javase/jdk17-archive-downloads.html). + +Make sure to set the `JAVA_HOME` environment variable to the location of your JDK installation. How to do this will be +dependent on your OS. ## Setting up the IDE -We recommend using IntelliJ IDEA for development to take advantage of the debugger, and provide instructions for it. -However, any IDE of your choice should work. -To get the free version of IntelliJ IDEA visit the [JetBrains website](https://www.jetbrains.com/idea/download/), and -download the Community Edition version (not Ultimate). It should be the second download button on the page. +We recommend using [IntelliJ IDEA](https://www.jetbrains.com/idea/download/) for development to take advantage of the +debugger, and provide instructions for it. However, any IDE of your choice should work. To set up IntelliJ, run: + ```shell ./gradlew idea ``` ### Disable wildcard imports + 1. In IntelliJ, click the gear icon in the top right and select settings 2. Go to Editor -> Code Style -> Java and select Imports from the tabs 3. Next to Scheme, select the gear icon and hit duplicate @@ -48,41 +45,46 @@ To set up IntelliJ, run: 6. Set `Names count to use static import with *` to 1000 ## Setting up your system + ### Set the maximum number of open files limit + There are many resources on the web for updating the limits of maximum number of open files for each operating system. We recommend setting a soft limit of at least `64000` and a hard limit of at least `524288`. Feel free to experiment with various values and find ones that work for you. ## Build the project + ```shell ./gradlew clean assemble ``` ## Run the test suite + ```shell ./gradlew check --continue ``` ## Gradle Cheatsheet + ```shell # Build project classes and artifacts ./gradlew assemble - + # Run all checks (spotbugs, code coverage, tests etc) ./gradlew check --continue - + # Run only enabled checks that may fail the build ./gradlew spotbugs - + # Run all checks, including ones that are in review/incubating stage, and do not fail the build ./gradlew spotbugs -Pspotallbugs -Pspotbugs.ignoreFailures - + # Run jacoco code coverage check, which will also generate jacoco report ./gradlew jacocoTestCoverageVerification # Run jacoco code coverage check along with jacoco report and diff coverage check ./gradlew jacocoTestCoverageVerification diffCoverage --continue - + # Run enabled checks only for main code and in da-vinci-client subproject ./gradlew :clients:da-vinci-client:spotbugsMain @@ -97,4 +99,4 @@ with various values and find ones that work for you. # To run a specific integration test ./gradlew :internal:venice-test-common:integrationTest --tests "fully.qualified.name.of.the.test" -``` \ No newline at end of file +``` diff --git a/docs/contributing/documentation/design-docs.md b/docs/contributing/documentation/design-docs.md new file mode 100644 index 00000000000..cc3a5d53400 --- /dev/null +++ b/docs/contributing/documentation/design-docs.md @@ -0,0 +1,63 @@ +# Design Documents + +## Venice Improvement Proposal (VIP) + +Venice follow the Venice Improvement Proposal (VIP). VIP is the mechanism used to propose changes to the Venice +Codebase. + +Not all the changes require a VIP treatment, Please use your commonsense to check if writing this VIP help yourself and +the reviewers time instead of just doing it in the pull request itself. Generally, this applies to + +- Large code refactors +- New functionality +- Public API changes + +In practical terms, the VIP defines a process in which developers can submit a design doc, receive feedback and get the +"go ahead" to execute. + +## Who can create a VIP? + +Any person willing to contribute to the Venice project is welcome to create a VIP. + +## How does the VIP process work? + +A VIP proposal can be in these states: + +1. **DRAFT**: (Optional) This might be used for contributors to collaborate and to seek feedback on an incomplete + version of the proposal. + +2. **DISCUSSION**: The proposal has been submitted to the community for feedback and approval. + +3. **ACCEPTED**: The proposal has been accepted by the Venice project. + +4. **REJECTED**: The proposal has not been accepted by the Venice project. + +5. **IMPLEMENTED**: The implementation of the proposed changes have been completed and everything has been merged. + +6. **RELEASED**: The proposed changes have been included in an official Venice release. + +The process works in the following way: + +1. The author(s) of the proposal will create a file named "VIP-xxx.md" in [proposal](../proposals/index.md) folder + cloning from the [template for VIP proposals](../proposals/vip-template.md). The "xxx" number should be chosen to be + the next number from the existing VIP issues, listed [here](../proposals/index.md) +2. The author(s) submit this file as a PR named "VIP-xxx: {short description}" in **DRAFT**/**DISCUSSION** stage. +3. People discuss using PR comments, each is its own threaded comment. General comments can be made as general comment + in the PR. There are two other ways for an interactive discussion. + 1. Venice Community [Slack Channel](http://slack.venicedb.org/), Create a slack channel with #VIP-xxx + 2. Venice Contributor Sync Meeting, see details [here](../contributing.md#community-syncup-meeting) at Contributor + Sync Meeting +4. Depending on the outcome of the discussion, the status could move to **ACCEPTED** or **REJECTED**, or it could stay + in **DISCUSSION** stage (e.g. if we agree tentatively on the broad strokes, but there are still action items to + refine certain aspects). At the end of this, the PR gets merged, and at that point the VIP will appear in the + directory of all historical VIPs. +5. If the merged VIP is still in **DISCUSSION** stage, then further PRs will be submitted to address the remaining + action items. +6. If the VIP is **ACCEPTED**, then next stages are implementation, with eventually some code change PR also carrying an + alteration of the VIP page to move the status to IMPLEMENTED. +7. All Pull Requests for this functionality, should prefix this "VIP-XXX" number in the title for quick access. + +## Acknowledgements + +This guide is inspired from the +[Apache Pulsar project proposal](https://github.com/apache/pulsar/blob/master/wiki/proposals/VIP.md) plan. diff --git a/docs/contributing/documentation/writing-docs.md b/docs/contributing/documentation/writing-docs.md new file mode 100644 index 00000000000..71ce9fd88ef --- /dev/null +++ b/docs/contributing/documentation/writing-docs.md @@ -0,0 +1,119 @@ +# Documentation Guideline + +We use GitHub Pages to host Venice documentation with MkDocs Material theme. Documentation is built automatically by +GitHub Actions when changes are pushed to the `main` branch. + +## General + +It is strongly encouraged that any code change which affects the validity of information in the docs also include +updates to the docs, so that both are kept in sync atomically. + +Experimental functionalities and future plans are also worth documenting, though they must be clearly marked as such, so +that users and operators reading those docs can make informed decisions about the level of risk they are willing to take +on if trying out a given functionality. If the level of maturity of a given functionality is not called out, then it +implicitly means that the functionality is considered mature and its API is unlikely to change. Undocumented configs and +APIs may or may not be considered mature and stable, and if in doubt, it is appropriate to open an Issue to request that +it be explicitly documented. + +In general, it is recommended to get familiar with the docs before writing more docs, to try to keep the style and +structure coherent. That being said, even if unsure where some documentation belongs, do err on the side of including it +(anywhere), and reviewers may suggest placing it elsewhere. + +## Hierarchy + +Documentation hierarchy is configured in `mkdocs.yml` in the `nav:` section. The navigation structure is defined +centrally: + +```yaml +nav: + - Home: README.md + - User Guide: + - user-guide/index.md + - Write APIs: + - user-guide/write-apis/batch-push.md +``` + +To add a new page: + +1. Create the markdown file in the appropriate directory +2. Add an entry to the `nav:` section in `mkdocs.yml` +3. The page title is taken from the first `# Heading` in the markdown file + +For more information, consult [MkDocs Material Documentation](https://squidfunk.github.io/mkdocs-material/). + +## Pictures and Diagrams + +It is encouraged to use diagrams within the documentation, but there are some guidelines to standardize the way it is +done, and to avoid certain anti-patterns. + +### Text-based Assets + +For text-based assets (which are preferred whenever feasible), we wish to check them into source control. This should +include both the displayable asset (e.g. in SVG format) and the source file from which the displayable asset was +generated (e.g. in XML format). This makes the docs self-contained, and enables contributors to edit the assets over +time. + +Diagrams conforming to these guidelines can be placed under the `/docs/assets/images` path of the repo, and then +embedded in the docs with a relative link like this: + +```markdown +![](../../assets/images/vip_3_read_path.drawio.svg) +``` + +The [draw.io](https://draw.io) service makes it easy to generate such assets. If using +[PlantUML](https://plantuml.com/starting), it's recommended to generate diagrams into svg format by following this +[guide](https://plantuml.com/svg). + +### Binary Assets + +For binary assets (e.g. PNG, BMP, JPG, etc.), we do NOT wish to check them into source control. Instead, they should be +linked from an external source. This can be done in GitHub itself. Within the Pull Request that proposes the doc change, +the contributor can insert images in the PR's description or comments, and then take the URL GitHub generated for it. +Then the modified files included in the PR can be edited to link to that image, and the PR updated. Externally hosted +images can be embedded with an absolute link like this: + +```markdown +![](https://user-images.githubusercontent.com/1248632/195111861-518f81c4-f226-4942-b88a-a34337da79e3.png) +``` + +## Emojis + +Here's a link to all the emojis available in README files: +[Emoji Cheat Sheet](https://github.com/ikatyang/emoji-cheat-sheet/blob/master/README.md). + +## Testing Doc Changes + +There are two ways to test doc changes, locally and on the public web. Local testing is convenient to iterate quickly, +while public web testing is useful to make sure that nothing breaks (e.g., especially if changing styles, Python +dependencies, or MkDocs configs) and to share more significant documentation changes with PR reviewers. + +### Testing Locally + +The docs are rendered and served by MkDocs with the Material theme. Install dependencies: + +```bash +pip install -r docs/doc-requirements.txt +``` + +Then from the repository root, run: + +```bash +mkdocs serve +``` + +Then navigate to [localhost:8000](http://localhost:8000) and view the docs in your browser. MkDocs hot reloads changes +to markdown files and configuration automatically. If you modify `mkdocs.yml`, the server will restart automatically. + +### Testing on the Public Web + +A GitHub fork can have its own documentation. This can be setup by: + +1. Navigating to the fork's Settings > Pages, i.e.: `https://github.com//venice/settings/pages` +2. Selecting which branch to publish the docs from +3. Selecting "Deploy from a branch" as the source (not "GitHub Actions") +4. Select branch "gh-pages" pointed at the root directory +5. Push changes to trigger the `deploy-docs.yml` workflow +6. Navigate to your fork's docs at: `https://.github.io/venice` + +For significant doc changes, please follow this process and add a link inside the PR to the docs hosted in the PR +author's own fork. diff --git a/docs/contributing/index.md b/docs/contributing/index.md new file mode 100644 index 00000000000..8fc62d0dbf4 --- /dev/null +++ b/docs/contributing/index.md @@ -0,0 +1,32 @@ +# Contributing to Venice + +Venice is an open source project. We welcome contributions! + +## Getting Started + +- [Code of Conduct](code-of-conduct.md) +- [Contributing Guide](contributing.md) +- [Security Policy](security.md) + +## Development + +- [Workspace Setup](development/workspace-setup.md) +- [Development Workflow](development/dev-workflow.md) +- [Testing Guide](development/testing.md) +- [Style Guide](development/style-guide.md) + +## Architecture + +- [Project Navigation](architecture/navigation.md) +- [Write Path](architecture/write-path.md) +- [Java Internals](architecture/java-internals.md) +- [Router API](architecture/router-api.md) + +## Documentation + +- [Writing Docs](documentation/writing-docs.md) +- [Design Docs](documentation/design-docs.md) + +## Proposals + +- [Venice Improvement Proposals (VIPs)](proposals/index.md) diff --git a/docs/contributing/proposals/index.md b/docs/contributing/proposals/index.md new file mode 100644 index 00000000000..62f540c52d1 --- /dev/null +++ b/docs/contributing/proposals/index.md @@ -0,0 +1,18 @@ +# Venice Improvement Proposals (VIPs) + +VIPs document significant design decisions and architectural changes. + +[VIP Process →](../documentation/design-docs.md) + +## Active VIPs + +| VIP | Title | Status | +| ----------------- | ----------------------------------- | ---------------- | +| [VIP-1](vip-1.md) | Authentication Service API | Accepted | +| [VIP-2](vip-2.md) | Removing Per Record Offset Metadata | Under Discussion | +| [VIP-3](vip-3.md) | Rust Server Read Path | Under Discussion | +| [VIP-4](vip-4.md) | Store Lifecycle Hooks | Accepted | +| [VIP-5](vip-5.md) | Facet Counting | Under Discussion | +| [VIP-6](vip-6.md) | Venice on Kubernetes | Under Discussion | + +[VIP Template →](vip-template.md) diff --git a/docs/contributing/proposals/vip-1.md b/docs/contributing/proposals/vip-1.md new file mode 100644 index 00000000000..56438077428 --- /dev/null +++ b/docs/contributing/proposals/vip-1.md @@ -0,0 +1,148 @@ +# VIP-1: Authentication Service API + +- **Status**: _Accepted_ +- **Author(s)**: _Enrico Olivelli_ +- **Pull Request**: [PR 471](https://github.com/linkedin/venice/pull/471) +- **Release**: N/A + +## Introduction + +Currently, Venice doesn't provide a well-defined extensible way to authenticate clients, but it only supports TLS based +authentication, and it is hard coded. This VIP proposes a new API to write plugins to authenticate clients. The first +follow-up work will be to implement a plugin to support JWT token based authentication. + +## Problem Statement + +Venice's services are based on HTTP/REST APIs, and we need a way to authenticate clients using standard mechanisms like +JWT tokens and OAuth2. It means that all the services must perform authentication (and authorization) checks on each +request. + +Therefore, we need a way to write plugins to perform authentication checks, this way it will be easy to add more and +more mechanisms in the future. + +Authentication mechanisms vary a lot from each other, and they are not only based on single steps; we need to support at +least only mechanisms that work well on HTTP and that can be performed in a single step, like basic HTTP Authentication, +that needs only to use an HTTP Header or TLS based Authentication, that is based on the client certificate exchanged +during the TLS handshake. + +## Scope + +1. **What is in scope?** + + - Define an API to write plugins to perform authentication checks on Venice components (controller, server and + router). + - The API must support single step authentication mechanisms (JWT, OAuth2, TLS client authentication). + - Support retrofitting the existing TLS mechanism as a plugin (without introducing the implementation). + - Ensure that the Authentication API is used by all the services and applied to every request. + - Ensure that the AuthorizerService is able to use the Authentication API to perform authorization checks. + +2. **What is out of scope?** + + - Remove legacy DynamicAccessController + - Implement a JWT plugin to support JWT token based authentication (this will be a follow-up work) + - Implement the TLS client certificate plugin + - Refactor the AuthorizerService APIs (even if some changes will be needed) + - Implement other authentication mechanisms (Kerberos, SAML, etc.) + - Modify the Admin Tools to support authentication (this will be a follow-up work) + - Deal with authentication against the PubSub broker (Kafka, Pulsar, etc.) + +## Project Justification + +This VIP is required in order to allow Venice user to use standard authentication mechanisms like JWT and OAuth2/OpenID +Connect. + +For instance JWT, OAuth2 and OpenId connect are widely used by Apache Pulsar users, and introducing these features will +help the adoption of Venice in the Pulsar community. + +## Functional specification + +The core of the VIP is the AuthenticationService API, that is used by all the services to perform authentication checks. + +The AuthenticationService API is a Java interface that mandates the contract for the authentication plugins. + +The goal of the AuthenticationService is to map an HTTP request to a user identity, a **Principal**, that can be used by +the AuthorizerService. + +As the AuthenticationService must work on all the VeniceComponents it won't have any hard dependency on the HTTP layer, +as in Venice we are using multiple technologies, depending on the Component. + +```java +public interface AuthenticationService extends Closeable { + /** + * Maps an HTTP request to a Principal. + * Any unchecked exception thrown by this method will be logged and the request will be rejected. + * @param requestAccessor access the HTTP Request fields + * @return the Principal or null if the request is not authenticated + */ + default Principal getPrincipalFromHttpRequest(HttpRequestAccessor requestAccessor); + + /** + * Generic Wrapper over an HTTP Request. + */ + interface HttpRequestAccessor { + String getHeader(String headerName); + X509Certificate getCertificate(); + } + + /** + * Lifecycle method, called when the Venice component is initialized. + * @param veniceProperties the configuration of the component being initialized + * @throws Exception + */ + default void initialise(VeniceProperties veniceProperties) throws Exception; + + /** + * Lifecycle method, called when the Venice component is closed. + */ + default void close(); +} + +``` + +You configure the classname of the AuthenticationService with an entry `authentication.service.class`. The +AuthenticationService is initialised by passing the VeniceProperties read from the configuration file. + +## Proposed Design + +Most the work is about introducing the new API and refactoring the existing code to use it. The legacy Authentication +mechanism will initially be left untouched, but when you configure the new AuthenticationService, the legacy mechanism +will be disabled. + +AuthenticationService and AuthorizerService will kick-in in spite of the legacy DynamicAccessController. And then the +AuthenticationService the ACL checks will be performed by the AuthorizerService, and not by the DynamicAccessController. + +In Venice clusters in which you configure AuthenticationService and AuthorizerService it is not expected that all the +current admin tools work, especially the ones that are based on the legacy DynamicAccessController (ACLs...). It is out +of the scope of this VIP to modify the admin tools. + +Multi-region support is not taken into account because we are only introducing a new API about Authentication, it +depends on every specific mechanism how to implement intra and inter region authentication. + +New Authentication mechanism may have an impact on performance depending on the technology user and the implementation. +It is out of the scope of this VIP to enter the details of the performance impact. It is possible that in the future in +order to support some authentication mechanisms we will need to introduce an asynchronous API to perform authentication +checks. + +## Development Milestones + +The implementation for this VIP introduces: + +- the Java API +- the Controller implementation (loading the plugin and calling the API) +- the Router implementation (loading the plugin and calling the API) +- the Server implementation (loading the plugin and calling the API) +- some dummy plugins to test the API + +## Test Plan + +The implementation will be tested with unit tests and integration tests, main topics: + +- AuthenticationService plugin lifecycle (boostrap, initialization, close) +- Verifying that the plugin is invoked by the Controller, Router and Server + +## References + +- [AuthenticationProvider API in Apache Pulsar](https://github.com/apache/pulsar/blob/master/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java) +- [Authentication Service Docs in Apache Pulsar](https://pulsar.apache.org/docs/3.0.x/security-authorization/) +- [OAuth2](https://oauth.net/2/) +- [JWT](https://jwt.io/introduction) diff --git a/docs/contributing/proposals/vip-2.md b/docs/contributing/proposals/vip-2.md new file mode 100644 index 00000000000..9139199b2e1 --- /dev/null +++ b/docs/contributing/proposals/vip-2.md @@ -0,0 +1,8 @@ +# VIP-2: Removing Per Record Offset Metadata From Venice-Server Storage With Heartbeats + +- **Status**: _Under Discussion_ +- **Author(s)**: _Zac Policzer_ +- **Pull Request**: [PR 513](https://github.com/linkedin/venice/pull/513) +- **Release**: _N/A_ + +Not yet merged. Discussed in [PR 513](https://github.com/linkedin/venice/pull/513). diff --git a/docs/proposals/vip-3.md b/docs/contributing/proposals/vip-3.md similarity index 84% rename from docs/proposals/vip-3.md rename to docs/contributing/proposals/vip-3.md index 2639af9675f..beb32fadac7 100644 --- a/docs/proposals/vip-3.md +++ b/docs/contributing/proposals/vip-3.md @@ -1,16 +1,9 @@ ---- -layout: default -title: VIP-3 Rust Server Read Path -parent: Proposals -permalink: /docs/proposals/vip-3 ---- - # VIP-3: Rust Server Read Path -* **Status**: _Under Discussion_ -* **Author(s)**: _Felix GV_ -* **Pull Request**: [PR 604](https://github.com/linkedin/venice/pull/604) -* **Release**: _N/A_ +- **Status**: _Under Discussion_ +- **Author(s)**: _Felix GV_ +- **Pull Request**: [PR 604](https://github.com/linkedin/venice/pull/604) +- **Release**: _N/A_ ## Introduction @@ -19,18 +12,18 @@ necessary to revisit past design choices and implementation details to consider level. This proposal is submitted in that spirit. The Fast Client is the most significant change to the read path since the inception of Venice, and it is progressing -well. It helps latency by eliminating the router hop and most importantly by eliminating an entire JVM from the hot +well. It helps latency by eliminating the router hop and most importantly by eliminating an entire JVM from the hot path. After this, the only JVM left in the backend is that of the server, which this proposal aims to eliminate as well. -The goal is to replace this Java code with another lower-level language which would be free of garbage collection, thus -making tail latency much more predictable. In addition, this would likely open up the door to increased throughput +The goal is to replace this Java code with another lower-level language which would be free of garbage collection, thus +making tail latency much more predictable. In addition, this would likely open up the door to increased throughput capacity as well. -The specific language proposed to replace Java is [Rust](https://www.rust-lang.org), which has become mature and well -accepted in the industry over the past years. It promises performance similar to C++, without some of its pitfalls, +The specific language proposed to replace Java is [Rust](https://www.rust-lang.org), which has become mature and well +accepted in the industry over the past years. It promises performance similar to C++, without some of its pitfalls, thanks to its borrow checker. -## Problem Statement +## Problem Statement The Venice read path should be completely free of garbage collection. Venice should be able to serve single get read requests, end-to-end (i.e. as measured by the client), at a p99.99 latency under 1 millisecond. @@ -60,15 +53,15 @@ Out of scope of the MVP is: The demand for AI use cases powered by Venice is growing on a steep trajectory along all dimensions, including the number of use cases, the queries per second, the keys per query, the number of records and the size of records. While -Venice is holding up for now and we can keep growing clusters and building new ones in the short-term, it is important -to also take a longer-term view and to kick off initiatives that will improve the foundation of the platform, even if -those initiatives will only land in production in the next 6-12 months. We need Venice to not only continue to further -scale horizontally, but to better scale vertically as well. As the footprint of the Venice backend continues to grow, +Venice is holding up for now and we can keep growing clusters and building new ones in the short-term, it is important +to also take a longer-term view and to kick off initiatives that will improve the foundation of the platform, even if +those initiatives will only land in production in the next 6-12 months. We need Venice to not only continue to further +scale horizontally, but to better scale vertically as well. As the footprint of the Venice backend continues to grow, the efficiency of each node in the system becomes more important to optimize. ## Functional specification -This is operator-facing work, so by design it should be as transparent as possible to the user. For the operator, the +This is operator-facing work, so by design it should be as transparent as possible to the user. For the operator, the new APIs and configs include: - New server config: @@ -76,8 +69,8 @@ new APIs and configs include: - To specify the port which the Rust server will listen on - Other configs of the Rust server implementation (e.g. worker thread count, etc; TBD) - Other config or store setting (TBD): - - To determine if cluster discovery will return the Rust server port or the regular port for a given store/cluster - (or possibly a mix, where only some servers enable the Rust implementation). + - To determine if cluster discovery will return the Rust server port or the regular port for a given store/cluster (or + possibly a mix, where only some servers enable the Rust implementation). From the user's perspective, as long as they are using the Fast Client with the gRPC mode enabled, they should be eligible to benefiting from the Rust server, if it happens to be enabled by the operator. @@ -90,31 +83,32 @@ normal, for both the ingestion path and the legacy read path. However, this Rust listen for gRPC requests, which it would handle and respond to. The diagram below shows the current and proposed architectures side-by-side. -![Architecture Diagram](../assets/images/vip_3_read_path.drawio.svg) +![Architecture Diagram](../../assets/images/vip_3_read_path.drawio.svg) ### Why Rust? In terms of a GC-free language, the main contender would be C++. We can certainly consider that option as well, but it -seems that nowadays Rust is broadly accepted as a solid alternative, and the momentum behind it is growing. Furthermore, -Rust's performance already rivals that of C++ in many workloads, and where there are differences one way or the other -they seem to be quite minor. The benefits of Rust's borrow checker are expected to reduce risk in terms of executing +seems that nowadays Rust is broadly accepted as a solid alternative, and the momentum behind it is growing. Furthermore, +Rust's performance already rivals that of C++ in many workloads, and where there are differences one way or the other +they seem to be quite minor. The benefits of Rust's borrow checker are expected to reduce risk in terms of executing this project and operating the end result safely. -In order to write idiomatic, higher quality, Rust code, the project will adopt [rustfmt](https://github.com/rust-lang/rustfmt) -and [rust-clippy](https://github.com/rust-lang/rust-clippy). +In order to write idiomatic, higher quality, Rust code, the project will adopt +[rustfmt](https://github.com/rust-lang/rustfmt) and [rust-clippy](https://github.com/rust-lang/rust-clippy). ### Why gRPC? -Because it seems easier than to re-implement Venice's bespoke [REST spec](../dev_guide/router_rest_spec.md). Also, -although gRPC is probably not perfect, it is likely more efficient than the current REST protocol, and will make it -easier to support clients in other languages. Also, we already have gRPC embedded in the stack following the Summer 2023 +Because it seems easier than to re-implement Venice's bespoke [REST spec](../architecture/router-api.md). Also, although +gRPC is probably not perfect, it is likely more efficient than the current REST protocol, and will make it easier to +support clients in other languages. Also, we already have gRPC embedded in the stack following the Summer 2023 internship project, so it might as well be leveraged. -There is more than one crate option for integrating with gRPC, but the most mature one seems to be [Tonic](https://crates.io/crates/tonic). +There is more than one crate option for integrating with gRPC, but the most mature one seems to be +[Tonic](https://crates.io/crates/tonic). ### Why JNI? -Because it is supported in Rust, and that should make the integration easier. An alternative would be to have the Java +Because it is supported in Rust, and that should make the integration easier. An alternative would be to have the Java server communicate with the Rust server via some other mechanism (loopback socket, pipes, etc) but then that would mean adding lots of forks in the code, and might also be less efficient. Using a non-JNI mechanism would also mean that just enabling the Rust server (even if it's not used yet) would already alter the behavior of the main code, thus creating @@ -152,29 +146,29 @@ Since the migration is expected to be controlled by the operator, and should onl Client (i.e. not for users that run a mix of Thin Clients and Fast Clients), the edge case solved by the more complicated solution 1 does not seem likely to come up. Therefore, the proposal is to pick either solution 2 or 3. -Also, it can be considered whether quota should be part of the MVP at all. If the Rust server mode will be tried in +Also, it can be considered whether quota should be part of the MVP at all. If the Rust server mode will be tried in dedicated clusters at first, then quota may not even be a must-have (though of course it will be needed eventually). ### Cluster Discovery -The general idea is that when the Rust mode is activated in a server, that server would announce that it has a gRPC +The general idea is that when the Rust mode is activated in a server, that server would announce that it has a gRPC endpoint at the port which the Rust server listens to. The announcement can be done by the Java server, to minimize complexity on the Rust side. Beyond that, there are many variants on how to make things work end-to-end. A few alternatives: -1. The Java and Rust servers are all announced to the same resource name. This means there is some gRPC resource which - a Fast Client can hit, and that could be served either by Java or Rust, and possibly both. Changing which path the - clients go through would require turning on or off the announcement on the Java and/or Rust side, which could be - done either via reconfiguring and bouncing the server, or via some admin tooling. +1. The Java and Rust servers are all announced to the same resource name. This means there is some gRPC resource which a + Fast Client can hit, and that could be served either by Java or Rust, and possibly both. Changing which path the + clients go through would require turning on or off the announcement on the Java and/or Rust side, which could be done + either via reconfiguring and bouncing the server, or via some admin tooling. 2. The Java and Rust servers are announced to different resource names. When cluster discovery happens, the client asks where to connect, the backend could decide to return one or the other of these resources, thus directing the traffic fully (at least for that one client instance) into one path or the other. Changing which path the clients go through - would require updating the mapping returned by cluster discovery, which would incur the same delay as cluster + would require updating the mapping returned by cluster discovery, which would incur the same delay as cluster migration. 3. Instead of announcing per-cluster resources, we could announce per-store resources, and then have some admin tooling to enable and disable the announcement for Rust and Java servers on a per-store basis. This could give the most - flexibility, since the client would not remain stuck with whatever resource name was returned by the cluster + flexibility, since the client would not remain stuck with whatever resource name was returned by the cluster discovery call done at startup time (and periodically thereafter), but rather could adapt rapidly. These options provide different levels of granularity of control and speed of change, with finer granularity and faster @@ -206,8 +200,8 @@ Similarly to auth/auth, there is already an abstraction for taking metrics out o metrics that will be gathered on the Rust side, two analogous approaches exist: build a Rust abstraction that can be implemented so that the Rust server gains the ability to emit metrics directly, or build a mechanism by which metrics held in Rust's memory can be copied over to the Java side, and emitted alongside the other Java side metrics. The latter -approach is analogous to what is being done in Ingestion Isolation, where the child process periodically sends all its -metrics to the parent process. For the same reason as in auth/auth that it would be simpler for the operator, the second +approach is analogous to what is being done in Ingestion Isolation, where the child process periodically sends all its +metrics to the parent process. For the same reason as in auth/auth that it would be simpler for the operator, the second approach is preferred here as well. ## Development Milestones @@ -216,7 +210,7 @@ The milestones for the MVP would be the following: 1. Build a simple RocksDB wrapper which gets packaged into a jar and exposes the same JNI APIs as RocksJava. This can be built with a single architecture at first. Print "hello world" when JNI is called successfully. -2. Swap RocksJava for the wrapper inside Venice. We can do a micro-benchmark at this stage to validate the assumption +2. Swap RocksJava for the wrapper inside Venice. We can do a micro-benchmark at this stage to validate the assumption that JNI via Rust is no worse than JNI via C++. 3. Integrate the chosen crate for adding gRPC support, and do a "hello world" of the RPC. 4. Add server configs to enable the Rust gRPC module (including basic service disco announcement). @@ -238,10 +232,10 @@ Other ideas for leveraging Rust, which are less likely to happen, but could even 1. Rust Fast Client which Java (and possibly other languages) could call via JNI (or similar), so that as much of the end-to-end transmission of data can be GC-free, with only the very last mile being done in the user app's language. -2. Rust Router. This is an alternative to the idea 1 above, in case we would want to keep the Thin Client as a part of +2. Rust Router. This is an alternative to the idea 1 above, in case we would want to keep the Thin Client as a part of the architecture in the long-term, perhaps as a way to implement support for clients in other languages more easily. 3. Rust ingestion. This would be quite complicated, since the Venice ingestion is very complex and sophisticated. It - could possibly be broken up into increments, e.g. we might start off with Write Compute being done in Rust, and + could possibly be broken up into increments, e.g. we might start off with Write Compute being done in Rust, and exposed as a single JNI call to Java (as opposed to the multiple calls it currently needs). 4. Rust controller. This is no doubt the least attractive milestone, since the controller is fairly complex, and not performance-sensitive. Almost certainly wouldn't happen. @@ -252,10 +246,10 @@ We of course need Rust unit tests for the Rust code. All Fast Client integration tests written in Java should get an extra permutation for enabling the Rust server. -Besides that, there will need to be extensive pre-prod certification to minimize the risk that enabling the Rust server +Besides that, there will need to be extensive pre-prod certification to minimize the risk that enabling the Rust server could destabilize the Venice server. -## References +## References Material for learning Rust: diff --git a/docs/contributing/proposals/vip-4.md b/docs/contributing/proposals/vip-4.md new file mode 100644 index 00000000000..9384b56eb0a --- /dev/null +++ b/docs/contributing/proposals/vip-4.md @@ -0,0 +1,270 @@ +# VIP-4: Store Lifecycle Hooks + +- **Status**: _Accepted_ +- **Author(s)**: _Felix GV_ +- **Pull Request**: [PR 881](https://github.com/linkedin/venice/pull/881) +- **Release**: _N/A_ + +## Introduction + +The [Venice Push Job](../../user-guide/write-apis/batch-push.md) takes data from a grid and pushes it to a store in all +regions. This works fine in many cases, but there are some use cases where we would like to have greater control over +the steps of the process. This proposal is to add new configs and hooks which can be used both to monitor and control +the push job in a finer-grained manner than is possible today. In particular, this proposal focuses on the way that each +individual region is handled. + +Currently, the sequence of steps happening within a push job is as follows: + +![Push Job Steps](../../assets/images/push_job_steps.drawio.svg) + +A few notes on the above diagram: + +- A new schema will be registered only if the data being pushed does not conform to any already known schema, and the + schema auto-registration config is enabled. If that config is disabled, then an unknown schema leads to job failure. + +- The compression job is optional, and whether it runs or not depends on configurations. + +- The data push job writes to a store-version pub sub topic in one of the regions (typically the one which is local to + where the push job is running), but all regions start ingesting right away, as soon as the data push job begins + writing. For regions that are remote from the topic the data push job is writing to, the leader servers are performing + replication, while in the region which contains that topic, the replication is a no-op. + +- The `SERVING` step is also called "version swap". If there are Da Vinci Clients, they will ingest and swap on their + own, and the child controller of that region will wait until all DVC instances have swapped (started serving) before + enacting the region-level swap, after which the servers will also start serving the new store-version to clients which + perform remote queries. That is why the "DVC Read Traffic SERVING" step is a dependency for the "Server Read Traffic + SERVING" step. + +## Problem Statement + +Pushing to all regions in parallel makes the push faster, but it also means that if the push causes an issue, the impact +is going to be global (affecting all regions). It would be desirable to have certain checks and balances that reduce the +blast radius in cases where the content of the push causes issues. Examples of such issues include: + +- Data quality issues: + - Incomplete data (due to issues in data generation logic, or in upstream datasets). + - Change in semantics (e.g. some embeddings trained by a new ML model / weights / params / etc. are incompatible with + that used at inference-time). +- Schema-related issues: + - Some optional fields which used to always be populated get deleted (or get populated with null) and the reading app + fails due to lack of null checking. This kind of app-side bug can happen even though a schema evolution is fully + compatible. +- Infra issues: + - Larger payloads take more resources, resulting in lack of capacity and thus latency degradation. + - Certain types of yet-unknown infra bugs are somehow triggered by a data push. + +Tighter control of how the new version of the dataset is deployed to each region could allow us to catch issues while +only one region is affected, and abort deploying to other regions. See Scope, below, for specific examples of flow +control strategies. + +In addition, we would like to make it easier to integrate the push job into proprietary monitoring systems such that +each region getting data deployed to it results in events getting emitted or other observability actions. + +## Scope + +This proposal is about full push jobs. Incremental pushes and nearline writes are out of scope. At the time of +submitting this proposal, it is undetermined whether this work will apply to stream reprocessing jobs. In terms of +priority, we care mostly about supporting full pushes from offline grids, and it may be fine to leave stream +reprocessing out of scope, although depending on the design details we choose, we may be able to support stream +reprocessing "for free" as well (i.e. if the hooks are executed in the controller). Incremental Push is out of scope of +this proposal. + +The goal is for lifecycle hooks to achieve the following use cases: + +- Orchestrate how data is served in each region, including: + - Ensuring a minimum delay (e.g. 1 hour) between each region beginning to serve the new store-version. + - Delaying the swapping of a new store-version to be within some time of day (e.g. during "business hours"). + - Performing custom health checks on the client applications to ensure that their key metrics are still healthy within + a region where a new store-version was swapped, before proceeding to more regions. Based on the outcome of this + check: + - Having the ability to abort the swapping of a store-version to further regions. + - Having the ability to rollback to the previous store-version in regions that already swapped. +- Trigger informational events in proprietary monitoring systems after important lifecycle milestones are completed. + +The above use cases all are operator-centric, and so (at least for now) there is no concern of making it very ergonomic +for Venice users to register new hooks or evolve old hooks dynamically. The general expectation is that there would be a +small number of hooks maintained by the Venice operators and that it's ok for hooks to be bundled and upgraded alongside +the Venice components. In cases where Venice users need to customize hook behaviors, that could be achieved via +store-level configs passed into hooks, and there is no need to provide the flexibility of letting users register whole +new hook implementations. + +## Project Justification + +The cost of having global impact in the case of issues mentioned above is too high, and we would like to provide +first-class options to reduce the blast radius. Building this within Venice itself will make it easier to automate these +methodologies, thus reducing toil for users. + +## Functional specification + +The proposed API for this functionality is described in code here: + +[//]: # "Got to remove the /venice prefix before pushing to main on linkedin/venice, otherwise links will be broken..." + +- **StoreLifecycleHooks**, which is the main part of this proposal. +- **StoreLifecycleEventOutcome**, which is the signal returned by some hooks to indicate that a given step should + proceed or abort. +- **StoreVersionLifecycleEventOutcome**, which is the signal returned by some other hooks which need more fine-grained + control over the workflow. In addition to proceeding and aborting, this also provides the option to wait, which tells + the hooks framework to try invoking the hook again later, and rollback, which tells the framework to rollback to the + previous store-version in all regions. +- **JobStatusQueryResponse**, which is the payload returned by the `/job` controller endpoint, is extended to include + status update timestamps. This will be populated by the child controller to indicate the time when its own individual + status last changed, and the parent controller will aggregate these into a map keyed by region. All hooks which return + the `StoreVersionLifecycleEventOutcome` will have access to this payload in their input, so that they can make + decisions based on the status of each region. The time when the status was last updated for a given region is useful + in order to achieve the use case of a hook which injects a delay between each region swap. The code change to support + these extra timestamps is included in this VIP, to demonstrate feasibility and present the proposed algorithm (see + `OfflinePushStatus::getStatusUpdateTimestamp`). + +## Proposed Design + +The main design consideration is where to execute the hooks. At a high level, there are three options: + +1. Within the push job. +2. Within the parent controller (chosen option). +3. Within the child controllers. + +It is also possible to consider invoking some hooks in one of these location while other hooks would be executed +elsewhere. The table below summarizes the feasibility and tradeoffs for each of the proposed hooks: + +| Hook function name | Actions | CC | PC | VPJ | +| -------------------------------------------: | :---------: | :-: | :-: | :-: | +| `validateHookConfig` | ➡️ ☠️ | ❌ | ✅ | ❌ | +| `preStartOfPushJob` | ➡️ ☠️ | ❌ | ✅ | ✅ | +| `postStartOfPushJob` | None | ❌ | ✅ | ✅ | +| `preSchemaRegistration` | ➡️ ☠️ | ❌ | ✅ | 1️⃣ | +| `postSchemaRegistration` | None | ✅ | ✅ | 1️⃣ | +| `preStoreVersionCreation` | ➡️ ☠️ ✋ ↩️ | ✅ | ✅ | ✅ | +| `postStoreVersionCreation` | None | ✅ | ✅ | ✅ | +| `preStartOfStoreVersionIngestionForDaVinci` | ➡️ ☠️ ✋ ↩️ | 2️⃣ | 2️⃣ | 2️⃣ | +| `postStartOfStoreVersionIngestionForDaVinci` | ➡️ ☠️ ✋ ↩️ | 2️⃣ | 2️⃣ | 2️⃣ | +| `postStoreVersionLeaderReplication` | None | ✅ | ❌ | ❌ | +| `preStoreVersionSwap` | ➡️ ☠️ ✋ ↩️ | ✅ | ✅ | ✅ | +| `postStoreVersionSwap` | ➡️ ☠️ ✋ ↩️ | ✅ | ✅ | ✅ | +| `preEndOfPushJob` | ➡️ ☠️ ✋ ↩️ | ✅ | ✅ | ✅ | +| `postEndOfPushJob` | None | ✅ | ✅ | ✅ | + +**Legend:** + +- The Actions column represent which control mechanism is available to each hook: + + - ➡️ Proceed: Move forward with this step. + - ☠️ Abort: Cancel this step (and as a consequence, short-circuit any future step that would come after this one). + - ✋ Wait: Let the hooks framework re-run this step later (i.e. 1 minute later, by default). + - ↩️ Rollback: Let the store go back to the store-version it had prior to beginning the push job (in all regions). + +- The last three columns are the feasibility of implementing this hook in a given component: + - **CC** Child Controller. + - **PC** Parent Controller. + - **VPJ** Venice Push Job. + - ✅ It is feasible to implement this hook within this component (without unreasonable complexity). + - ❌ It is NOT feasible to implement this hook within this component (without unreasonable complexity). + - 1️⃣ Schema registration hooks in push jobs could only be invoked in cases where auto-registration is enabled and the + new schema originates from the push job itself, whereas schema registrations which are performed directly on the + controller could not trigger the hook. + - 2️⃣ The hook for the start of ingestion for Da Vinci Clients is tricky for a few reasons. The start of ingestion is + controlled by updating the Meta Store, which is a non-replicated system store updated by child controllers, so those + must be involved (either by running the hook there in the first place, or by having some mechanism that enables the + parent or VPJ to inform the child controllers of when the system store is eligible for getting updated, such as by + adding a new field to the `AddVersion` admin channel command). However, see Rollback Support below for why running + hooks in the child controller may be insufficient. + +### Rollback Support + +In order to support the ability for the hooks which return the `StoreVersionLifecycleEventOutcome` to rollback, the most +natural way to achieve this is likely to involve the parent controller, either by having those hooks run there in the +first place, or by having a propagation mechanism to it: + +- If the hooks are executed in the child controller, the propagation mechanism might be to extend the job status check + which the parent controller does periodically in order for the child to inform the parent of the need to rollback, or + else build a new mechanism for the child to directly interact with the parent (or even with other child controllers + directly...). + +- If the hooks are executed in VPJ, then it would need to interact with the parent via the controller client to trigger + the rollback. + +### Configs + +There needs to be new configs: + +- `venice.store.lifecycle.hooks`: Comma-separated list of FQCN of the hook implementations to load. + +- `venice.store.lifecycle.hooks.threads`: Number of threads used by the hooks framework to execute all hooks. + +- `venice.store.lifecycle.hooks.timeout.seconds`: Max duration allowed for a hook before the hooks framework interrupts + it. + +- `venice.store.lifecycle.hooks.wait.interval.seconds`: The time to wait before re-invoking a hook which returned `WAIT` + (default: 60 seconds). + +- `venice.store.lifecycle.hooks.configs.`: Any number of configs to be passed (after clipping everything + before the `` part) into the hooks constructor. + +In addition, the store config will get a new `Map` of store-level config overrides. Those configs are +"[stringly-typed](https://wiki.c2.com/?StringlyTyped)", rather than strongly-typed, since we are not aware of the +configs needed by each hook at compile-time, and we therefore cannot shape the definition of the store config schema +accordingly. This issue is mitigated via the `validateHookConfig`, which can be used to prevent invalid configs from +entering the system. + +### Metrics + +There needs to be new metrics to monitor hook health. Each new metric will be per function and per registered hooks +class (i.e. 13 functions per class if we implement all of them, or less if we cut the scope). For each class/function +hook, there will be: + +- the occurrence rate of: + - hook invocations + - each hook return signal (for the functions that return something other than `void`) + - failures (exceptions) + - timeouts +- the time spend waiting in queue before being executed (which will be useful to determine if the thread pool count is + under-provisioned) + +N.B.: Implementing metrics from within VPJ is a bit more complicated, whereas controllers already have the ability to +emit metrics. + +### Design Recommendation + +Running hooks in the parent controller is probably most straightforward. The only issue is the inability to support the +`postStoreVersionLeaderReplication` hook, but that one is not critical and could be left out of scope. + +Concretely, choosing the parent controller path would work like this: + +1. The parent controller would invoke the hooks for `preStartOfStoreVersionIngestionForDaVinci` and for + `preStoreVersionCreation` for each of the regions. If all hooks respond with `PROCEED` then it's essentially a normal + push, otherwise it would configure the `AddVersion` command sent to the admin channel with the appropriate inclusions + in `targetedRegions` (depending on the result of `preStoreVersionCreation`) and in a new field for controlling the + DVC ingestion (depending on the result of `preStartOfStoreVersionIngestionForDaVinci`) which the child controller + would honor by holding off on writing to the Meta Store. + +2. The parent controller would create the new store-version with `versionSwapDeferred = true`. + +3. When the parent controller sees that a region has completed ingestion, it would invoke the `preStoreVersionSwap` hook + for that region, and if the hook responds with `PROCEED`, then it would send an admin command targeted for that + region to swap the current version. + +4. If at any stage the hooks respond with `ROLLBACK` then the parent controller would send more admin channel commands + to do the swap in the reverse direction. + +**This recommendation has been accepted, after design reviews, hence we will implement hooks in the parent controller.** + +## Development Milestones + +At a high-level: + +1. Evolve the admin channel protocol. +2. Implement the child controller changes for dealing with the admin channel changes. +3. Implement the parent controller changes to support the hooks framework and the orchestration described above. + +More fine-grained plan TBD after finalizing the design. + +## Test Plan + +The first hooks will be built such that they are no-op by default, and require a hook config to enable them. That hook +config will be left disabled in the global config, and will be tested at small scale via the store-level overrides. + +After store-level testing and stabilization is satisfactory, we will begin enabling them globally. + +## References + +N/A. diff --git a/docs/proposals/vip-5.md b/docs/contributing/proposals/vip-5.md similarity index 86% rename from docs/proposals/vip-5.md rename to docs/contributing/proposals/vip-5.md index 1fcc066e0d7..05a79725f84 100644 --- a/docs/proposals/vip-5.md +++ b/docs/contributing/proposals/vip-5.md @@ -1,37 +1,30 @@ ---- -layout: default -title: VIP-5 Facet Counting -parent: Proposals -permalink: /docs/proposals/vip-5 ---- - # VIP-5: Facet Counting -* **Status**: _Accepted_ -* **Author(s)**: _Felix GV_ -* **Pull Request**: [PR 1612](https://github.com/linkedin/venice/pull/1612) -* **Release**: _N/A_ +- **Status**: _Accepted_ +- **Author(s)**: _Felix GV_ +- **Pull Request**: [PR 1612](https://github.com/linkedin/venice/pull/1612) +- **Release**: _N/A_ ## Introduction -Facet Counting is a type of aggregation query popular in the search domain. It provides information about the +Facet Counting is a type of aggregation query popular in the search domain. It provides information about the cardinality of values for the fields of documents of interest. Given that search is one of the types of derived data workloads in which Venice already participates (i.e., the Da Vinci -Client can serve as a search solution's forward index component), there is demand from Venice users to increase the +Client can serve as a search solution's forward index component), there is demand from Venice users to increase the scope of capabilities such that Facet Counting can also be performed natively by Venice. -This document introduces the domain, and suggests a roadmap for implementing this capability in various phases, +This document introduces the domain, and suggests a roadmap for implementing this capability in various phases, including on the client-side and server-side. -## Problem Statement +## Problem Statement This section dives deeper in defining what Facet Counting is, and how to integrate it into Venice. -Before diving into specifics, it may be useful to provide an analogy in order to make a general observation: _the +Before diving into specifics, it may be useful to provide an analogy in order to make a general observation: _the proposal in this VIP has similarities and differences compared to Read Compute_. Read Compute, as it exists today, is a record-wise (or we could say: row-wise) operation, meaning that for a given input record there is exactly one output -record. This type of workload is very natural to push down into the server-side, in such way that the work is split +record. This type of workload is very natural to push down into the server-side, in such way that the work is split across many servers and the client can retrieve the various output records individually. The use cases presented here also have a portion of work which is executable on a per-record basis, and therefore has the potential of being pushed down to the server-side, however, given that they are "aggregation queries", there is also a final processing step which @@ -39,11 +32,12 @@ must be performed in some central location (e.g., in the client). The work can t ### Facet Counting Use Cases -Let's define what Facet Counting is, with a series of examples from simple to more complex, using SQL to explain it +Let's define what Facet Counting is, with a series of examples from simple to more complex, using SQL to explain it (although SQL is just a convenient way to express query semantics, but is not part of this proposal). -These are functioning SQL queries that have been run on a DuckDB database populated by Venice's own Push Job Details -[system store](../ops_guide/system_stores.md). This system store's Avro schema can be seen here: [key](https://github.com/linkedin/venice/blob/main/internal/venice-common/src/main/resources/avro/PushJobStatusRecordKey/v1/PushJobStatusRecordKey.avsc), +These are functioning SQL queries that have been run on a DuckDB database populated by Venice's own Push Job Details +[system store](../../operations/data-management/system-stores.md). This system store's Avro schema can be seen here: +[key](https://github.com/linkedin/venice/blob/main/internal/venice-common/src/main/resources/avro/PushJobStatusRecordKey/v1/PushJobStatusRecordKey.avsc), [value](https://github.com/linkedin/venice/blob/main/internal/venice-common/src/main/resources/avro/PushJobDetails/v4/PushJobDetails.avsc). The SQL schema for the table is also included below, though only a subset of these columns are used in the examples: @@ -102,8 +96,8 @@ LIMIT 5; #### Facet Counting by Buckets Within a Column -Another example would be to group not by values, but by buckets. An example of this would be to have buckets for -"last 24h", "last week", "last 30 days". +Another example would be to group not by values, but by buckets. An example of this would be to have buckets for "last +24h", "last week", "last 30 days". ```sql SET VARIABLE most_recent_job_time = (SELECT MAX(reportTimestamp) FROM current_version); @@ -142,27 +136,27 @@ ORDER BY cnt; #### Facet Counting on Multiple Columns -Finally, a common example of Facet Counting would be to perform the same as above, but for multiple columns, all at -once. In real scenarios, there could be hundreds of columns included, but to keep things simple we will do only two +Finally, a common example of Facet Counting would be to perform the same as above, but for multiple columns, all at +once. In real scenarios, there could be hundreds of columns included, but to keep things simple we will do only two columns counted by values and one column counted by bucket: ```sql SELECT * FROM ( SELECT 'clusterName' AS col, - clusterName AS value_or_bucket, - COUNT(clusterName) AS cnt - FROM current_version - GROUP BY clusterName - ORDER BY cnt - DESC LIMIT 5) + clusterName AS value_or_bucket, + COUNT(clusterName) AS cnt + FROM current_version + GROUP BY clusterName + ORDER BY cnt + DESC LIMIT 5) UNION SELECT * FROM ( - SELECT 'storeName' AS col, - storeName AS value_or_bucket, - COUNT(storeName) AS cnt - FROM current_version - GROUP BY storeName - ORDER BY cnt + SELECT 'storeName' AS col, + storeName AS value_or_bucket, + COUNT(storeName) AS cnt + FROM current_version + GROUP BY storeName + ORDER BY cnt DESC LIMIT 5) UNION SELECT * FROM ( @@ -211,13 +205,13 @@ ORDER BY col, cnt DESC; #### Filtering The above examples all demonstrate Facet Counting being performed on a full table. And while that is feasible, it can be -a costly proposition. In practice, a more common scenario is to perform Facet Counting on a subset of rows of the +a costly proposition. In practice, a more common scenario is to perform Facet Counting on a subset of rows of the dataset. In the case of this proposal, the goal is to provide the ability to perform Facet Counting on some pre-defined keys within the Venice dataset. ### Where to Perform the Counting? -There are a variety of locations where the counting could be performed: client, router, server. Ultimately, it is +There are a variety of locations where the counting could be performed: client, router, server. Ultimately, it is probably ideal to have the ability to perform it in all three locations, and to decide by configuration what mode the system will operate in, so that we have the most operational flexibility. That, however, does not mean that we need to implement all of these in order to start getting value from the project (see Development Milestones). @@ -225,33 +219,33 @@ implement all of these in order to start getting value from the project (see Dev #### Client-side Computation There is already support for performing Read Compute on the client-side, which is useful as a fallback in cases where -server-side Read Compute is disabled for the queried store. Similarly, the ability to perform server-side Facet Counting -should be enabled via a store config, and the client should be capable of gracefully degrading to client-side compute -if the server-side is disabled. This is important so that users can make use of the API no matter what the server-side +server-side Read Compute is disabled for the queried store. Similarly, the ability to perform server-side Facet Counting +should be enabled via a store config, and the client should be capable of gracefully degrading to client-side compute if +the server-side is disabled. This is important so that users can make use of the API no matter what the server-side settings are, and configs then become a lever for shifting work across components, rather than a functional blocker. #### Server-side Computation The appeal of supporting server-side computations for Facet Counting is the same as for Read Compute: -1. The response sizes should be much smaller, thus saving on network bandwidth and improving the overall end-to-end +1. The response sizes should be much smaller, thus saving on network bandwidth and improving the overall end-to-end performance. -2. Moreover, the work can be partitioned across many servers, each of which would need to do just a fraction of the +2. Moreover, the work can be partitioned across many servers, each of which would need to do just a fraction of the total. The difference with currently supported Read Compute queries is that, given that Facet Counting is an aggregation query, there still needs to be some amount of work performed in the component which receives the subset of results from each -server. In the case of the Fast Client, this final step must be performed on the FC itself. In the case of the Thin +server. In the case of the Fast Client, this final step must be performed on the FC itself. In the case of the Thin Client, it could be performed either on the TC or on the Router. #### Router-side Computation -As mentioned above, for Thin Clients performing Facet Counting queries on a store where server-side computation is -enabled, the final step of the query processing could be done either on the client-side or router-side. From a +As mentioned above, for Thin Clients performing Facet Counting queries on a store where server-side computation is +enabled, the final step of the query processing could be done either on the client-side or router-side. From a functional standpoint, either way works, but from an efficiency standpoint, it may be better to do it on the router-side -to further reduce network bandwidth (between router and client). That being said, given that latency-sensitive +to further reduce network bandwidth (between router and client). That being said, given that latency-sensitive applications ought to onboard to the Fast Client anyway, there may be diminishing returns in expending effort to support -router-side computation. Therefore, this milestone can be scheduled last, and it may be acceptable to not implement it +router-side computation. Therefore, this milestone can be scheduled last, and it may be acceptable to not implement it at all. ## Scope @@ -272,7 +266,7 @@ The following is out of scope: ## Project Justification The goal of this project is to make Venice more amenable to leverage in a variety of derived data use cases. This can -help users minimize the complexity of juggling multiple independent systems, leading to productivity and operability +help users minimize the complexity of juggling multiple independent systems, leading to productivity and operability improvements for them. While it is already possible for users to manually implement Facet Counting on data retrieved via Batch Get (which would @@ -309,9 +303,11 @@ public interface ComputeAggregationRequestBuilder extends ExecutableRequestBu * @param The type of the fields to apply the bucket predicates to. */ ComputeAggregationRequestBuilder countGroupByBucket( - Map> bucketNameToPredicate, - String... fieldNames); + Map> bucketNameToPredicate, + String... fieldNames + ); } + ``` Responses could be accessed using the following container: @@ -330,15 +326,17 @@ public interface ComputeAggregationResponse { */ Map getBucketNameToCount(String fieldName); } + ``` Using the proposed API to achieve the `Facet Counting on Multiple Columns` use case above would look like this: ```java public class VIP5Example { + public void vip(AvroGenericReadComputeStoreClient client, Set keySet) { // Note that the computeAggregation() API is not added in this PR, and will be added once initial support is built - ComputeAggregationRequestBuilder requestBuilder = client.computeAggregation(); + ComputeAggregationRequestBuilder requestBuilder = client.computeAggregation(); // Using the Predicate API to define the filtering criteria of each bucket long currentTime = System.currentTimeMillis(); @@ -346,30 +344,31 @@ public class VIP5Example { bucketByTimeRanges.put("last_24h", LongPredicate.greaterThan(currentTime - 1 * Time.MS_PER_DAY)); bucketByTimeRanges.put("last_week", LongPredicate.greaterThan(currentTime - 7 * Time.MS_PER_DAY)); bucketByTimeRanges.put("last_30_days", LongPredicate.greaterThan(currentTime - 30 * Time.MS_PER_DAY)); - + // Facet count for multiple columns, including both grouped by value and grouped by buckets - ComputeAggregationRequestBuilder requestBuilder = requestBuilder - .countGroupByValue(5, "clusterName", "storeName") - .countGroupByBucket(bucketByTimeRanges, "reportTimestamp"); + ComputeAggregationRequestBuilder requestBuilder = requestBuilder + .countGroupByValue(5, "clusterName", "storeName") + .countGroupByBucket(bucketByTimeRanges, "reportTimestamp"); // Specify filter to specific keys in order to execute the query CompletableFuture facetCountResponse = requestBuilder.execute(keySet); } } + ``` -As we can see, the proposed DSL is significantly more succinct than the equivalent SQL. That is because although SQL can +As we can see, the proposed DSL is significantly more succinct than the equivalent SQL. That is because although SQL can be made to do nearly anything, Facet Counting of multiple columns in a single query is not a common use case. ## Proposed Design -The client-side computation is very similar to Read Compute and therefore fairly straightforward. Essentially, the +The client-side computation is very similar to Read Compute and therefore fairly straightforward. Essentially, the client would retrieve the values for the keys of interest via Batch Get, then perform the computation locally. Regarding server-side computation, it could be achieved either by extending the current `/compute` endpoint on servers, or as a new endpoint. Given that there are significant differences between the two, it may be tedious to cram the two of -them into the same endpoint. Moreover, since there is no need for the time being to combine Read Compute and Facet -Counting for the same set of keys as part of a single query, there is no incentive for now to incur this complexity +them into the same endpoint. Moreover, since there is no need for the time being to combine Read Compute and Facet +Counting for the same set of keys as part of a single query, there is no incentive for now to incur this complexity cost. If, in the future, it becomes necessary to combine these types of queries together, then the architecture can be further evolved at that point. @@ -378,18 +377,18 @@ The Facet Counting endpoint would need to have its own wire protocol, similar to 1. Start by listing the computation details (encoded from what was specified in `ComputeAggregationRequestBuilder`). 2. Then list all the keys to query and compute on. -It is necessary to support protocol evolution, but being separate from Read Compute, these two can evolve on separate +It is necessary to support protocol evolution, but being separate from Read Compute, these two can evolve on separate tracks. ## Development Milestones -1. Start with client-side support, so that users can start using the API as soon as possible, and that it works +1. Start with client-side support, so that users can start using the API as soon as possible, and that it works regardless of store and server settings. 2. Then implement the wire protocol and server-side support. ### Future Work -Although this is out of scope from the current proposal, it is interesting to note that having the scaffolding in place +Although this is out of scope from the current proposal, it is interesting to note that having the scaffolding in place to perform aggregation queries opens up the door to performing other kinds of aggregations besides counting, e.g., min, max, average, sum, or other functions... @@ -398,10 +397,10 @@ max, average, sum, or other functions... This functionality requires the full scope of test methodologies: unit and integration tests, followed by collaboration with early adopter users to integrate and gradually ramp. -## References +## References 1. [Facet Counting in Lucene](https://lucene.apache.org/core/4_1_0/facet/org/apache/lucene/facet/doc-files/userguide.html#facet_features) ## Appendix -See the code attached to [PR 1612](https://github.com/linkedin/venice/pull/1612). \ No newline at end of file +See the code attached to [PR 1612](https://github.com/linkedin/venice/pull/1612). diff --git a/docs/proposals/vip-6.md b/docs/contributing/proposals/vip-6.md similarity index 51% rename from docs/proposals/vip-6.md rename to docs/contributing/proposals/vip-6.md index a39ca274a12..82881483b77 100644 --- a/docs/proposals/vip-6.md +++ b/docs/contributing/proposals/vip-6.md @@ -1,37 +1,29 @@ ---- -layout: default -title: VIP-6 Venice on Kubernetes -parent: Proposals -permalink: /docs/proposals/vip-6 ---- - # VIP-6: Run Venice on Kubernetes -* **Status**: _Under Discussion_ -* **Author(s)**: _Nisarg Thakkar_ -* **Pull Request**: [PR 2064](https://github.com/linkedin/venice/pull/2064) -* **Release**: _N/A_ +- **Status**: _Under Discussion_ +- **Author(s)**: _Nisarg Thakkar_ +- **Pull Request**: [PR 2064](https://github.com/linkedin/venice/pull/2064) +- **Release**: _N/A_ -_Please make a copy of this page - DO NOT EDIT this design document directly, unless you are making -changes to the template._ +_Please make a copy of this page - DO NOT EDIT this design document directly, unless you are making changes to the +template._ _Remove the instructions (in italics) before publishing._ -## Introduction _(200 words at most)_ - -_This section should provide a concise and comprehensive overview of this document. After reading the introduction, -the reader should have a high level understanding of the problem being solved (the "what"), why the problem is -important (the "why"), and how this design intends to solve it (the "how"). -When the draft design document is circulated, readers may use the introduction to determine if it is -applicable to their area of work and if they are interested in reading further._ +## Introduction _(200 words at most)_ -## Problem Statement +_This section should provide a concise and comprehensive overview of this document. After reading the introduction, the +reader should have a high level understanding of the problem being solved (the "what"), why the problem is important +(the "why"), and how this design intends to solve it (the "how"). When the draft design document is circulated, readers +may use the introduction to determine if it is applicable to their area of work and if they are interested in reading +further._ -_This section should frame the problem clearly and state the non-negotiable requirements for a solution. -There is some overlap with the Introduction section but it is here for emphasis since it forms the basis -for evaluating any proposed design. There should be no disagreement on the problem statement, -otherwise a design or implementation is unlikely to be successful._ +## Problem Statement +_This section should frame the problem clearly and state the non-negotiable requirements for a solution. There is some +overlap with the Introduction section but it is here for emphasis since it forms the basis for evaluating any proposed +design. There should be no disagreement on the problem statement, otherwise a design or implementation is unlikely to be +successful._ ## Scope @@ -43,12 +35,13 @@ _Call out explicitly:_ 2. **What is out of scope?** - _For example, this functionality won’t apply to DaVinci Clients._ + _For example, this functionality won’t apply to DaVinci Clients._ ## Project Justification -_This section should justify the necessity to address the problem mentioned above, in another way, -we could think about what will happen if we don’t solve the problem, and we could consider the following dimensions._ +_This section should justify the necessity to address the problem mentioned above, in another way, we could think about +what will happen if we don’t solve the problem, and we could consider the following dimensions._ + 1. New use cases. 2. Customer experience 3. Productivity improvement. @@ -65,28 +58,29 @@ we could think about what will happen if we don’t solve the problem, and we co ## Functional Specification -_If this is a development of a functionality/product for users, specify what it should look like from the user's point of view. -It may include:_ +_If this is a development of a functionality/product for users, specify what it should look like from the user's point +of view. It may include:_ + 1. Public API. 2. Expected public behavior. 3. Public behavior of error handling, etc. ## Proposed Design -_This section must describe the proposed design and implementation in detail, and how it -addresses the problems outlined in the problem statement. It must provide details on alternative solutions -that were considered, stating clearly why they lost out to the final design. This latter point is particularly -important or else we tend to vacillate on old decisions because we cannot remember the original rationale. -Phrase this section in terms of trade offs since it is rare that one approach is better in all ways; there is -generally a tradeoff and making that explicit makes it much easier to understand._ +_This section must describe the proposed design and implementation in detail, and how it addresses the problems outlined +in the problem statement. It must provide details on alternative solutions that were considered, stating clearly why +they lost out to the final design. This latter point is particularly important or else we tend to vacillate on old +decisions because we cannot remember the original rationale. Phrase this section in terms of trade offs since it is rare +that one approach is better in all ways; there is generally a tradeoff and making that explicit makes it much easier to +understand._ _The following aspects of the design must be covered when applicable:_ 1. _Changes to APIs or protocols or data format highlighting any compatibility issues and how they will be addressed._ 2. _Major components and sequence diagrams between them._ 3. _Multi-region implications including Parent/Child Controller communication._ -4. _Alternative designs considered, and brief explanation of tradeoffs and decision versus the chosen design, -and important aspects to consider_: +4. _Alternative designs considered, and brief explanation of tradeoffs and decision versus the chosen design, and + important aspects to consider_: 1. Security. 2. Performance. 3. Maintainability. @@ -102,34 +96,24 @@ and important aspects to consider_: 1. _Conclusions/Decisions made in the design review sessions. Explanation why it’s selected_ ## Development Milestones -_This section described milestones for rather large projects so that we can set up intermediate goals for -better progress management. -For small projects, this may not be necessary._ + +_This section described milestones for rather large projects so that we can set up intermediate goals for better +progress management. For small projects, this may not be necessary._ ## Test Plan -_Describe in few sentences how the functionality will be tested. - How will we know that the implementation works as expected? How will we know nothing broke?_ +_Describe in few sentences how the functionality will be tested. How will we know that the implementation works as +expected? How will we know nothing broke?_ 1. _What unit tests would be added to cover the critical logic?_ 2. _What integration tests would be added to cover major components, if they cannot be tested by unit tests?_ 3. _Any tests to cover performance or Security concerns._ - -## References +## References _List any references here._ ## Appendix -_Add supporting information here, such as the results of performance studies and back-of-the-envelope calculations. +_Add supporting information here, such as the results of performance studies and back-of-the-envelope calculations. Organize it however you want but make sure it is comprehensible to your readers._ - - - - - - - - - diff --git a/docs/proposals/VIP_TEMPLATE.md b/docs/contributing/proposals/vip-template.md similarity index 50% rename from docs/proposals/VIP_TEMPLATE.md rename to docs/contributing/proposals/vip-template.md index 363918e2d13..e40c5ca2ea1 100644 --- a/docs/proposals/VIP_TEMPLATE.md +++ b/docs/contributing/proposals/vip-template.md @@ -1,37 +1,29 @@ ---- -layout: default -title: VIP Template # Replace with: VIP-{VIP_Number} {VIP_Title} -parent: Proposals -permalink: /docs/proposals/vip_template # Replace with: /docs/proposals/vip-{VIP_Number} ---- - # VIP-`$VIP_Number`: `$VIP_Title` -* **Status**: _"one of ['Under Discussion', 'Accepted', 'Implemented', 'Rejected']"_ -* **Author(s)**: _(Names)_ -* **Pull Request**: _(Link to the main pull request to resolve this VIP)_ -* **Release**: _(Which release include this VIP)_ +- **Status**: _"one of ['Under Discussion', 'Accepted', 'Implemented', 'Rejected']"_ +- **Author(s)**: _(Names)_ +- **Pull Request**: _(Link to the main pull request to resolve this VIP)_ +- **Release**: _(Which release include this VIP)_ -_Please make a copy of this page - DO NOT EDIT this design document directly, unless you are making -changes to the template._ +_Please make a copy of this page - DO NOT EDIT this design document directly, unless you are making changes to the +template._ _Remove the instructions (in italics) before publishing._ -## Introduction _(200 words at most)_ - -_This section should provide a concise and comprehensive overview of this document. After reading the introduction, -the reader should have a high level understanding of the problem being solved (the "what"), why the problem is -important (the "why"), and how this design intends to solve it (the "how"). -When the draft design document is circulated, readers may use the introduction to determine if it is -applicable to their area of work and if they are interested in reading further._ +## Introduction _(200 words at most)_ -## Problem Statement +_This section should provide a concise and comprehensive overview of this document. After reading the introduction, the +reader should have a high level understanding of the problem being solved (the "what"), why the problem is important +(the "why"), and how this design intends to solve it (the "how"). When the draft design document is circulated, readers +may use the introduction to determine if it is applicable to their area of work and if they are interested in reading +further._ -_This section should frame the problem clearly and state the non-negotiable requirements for a solution. -There is some overlap with the Introduction section but it is here for emphasis since it forms the basis -for evaluating any proposed design. There should be no disagreement on the problem statement, -otherwise a design or implementation is unlikely to be successful._ +## Problem Statement +_This section should frame the problem clearly and state the non-negotiable requirements for a solution. There is some +overlap with the Introduction section but it is here for emphasis since it forms the basis for evaluating any proposed +design. There should be no disagreement on the problem statement, otherwise a design or implementation is unlikely to be +successful._ ## Scope @@ -43,12 +35,13 @@ _Call out explicitly:_ 2. **What is out of scope?** - _For example, this functionality won’t apply to DaVinci Clients._ + _For example, this functionality won’t apply to DaVinci Clients._ ## Project Justification -_This section should justify the necessity to address the problem mentioned above, in another way, -we could think about what will happen if we don’t solve the problem, and we could consider the following dimensions._ +_This section should justify the necessity to address the problem mentioned above, in another way, we could think about +what will happen if we don’t solve the problem, and we could consider the following dimensions._ + 1. New use cases. 2. Customer experience 3. Productivity improvement. @@ -65,28 +58,29 @@ we could think about what will happen if we don’t solve the problem, and we co ## Functional Specification -_If this is a development of a functionality/product for users, specify what it should look like from the user's point of view. -It may include:_ +_If this is a development of a functionality/product for users, specify what it should look like from the user's point +of view. It may include:_ + 1. Public API. 2. Expected public behavior. 3. Public behavior of error handling, etc. ## Proposed Design -_This section must describe the proposed design and implementation in detail, and how it -addresses the problems outlined in the problem statement. It must provide details on alternative solutions -that were considered, stating clearly why they lost out to the final design. This latter point is particularly -important or else we tend to vacillate on old decisions because we cannot remember the original rationale. -Phrase this section in terms of trade offs since it is rare that one approach is better in all ways; there is -generally a tradeoff and making that explicit makes it much easier to understand._ +_This section must describe the proposed design and implementation in detail, and how it addresses the problems outlined +in the problem statement. It must provide details on alternative solutions that were considered, stating clearly why +they lost out to the final design. This latter point is particularly important or else we tend to vacillate on old +decisions because we cannot remember the original rationale. Phrase this section in terms of trade offs since it is rare +that one approach is better in all ways; there is generally a tradeoff and making that explicit makes it much easier to +understand._ _The following aspects of the design must be covered when applicable:_ 1. _Changes to APIs or protocols or data format highlighting any compatibility issues and how they will be addressed._ 2. _Major components and sequence diagrams between them._ 3. _Multi-region implications including Parent/Child Controller communication._ -4. _Alternative designs considered, and brief explanation of tradeoffs and decision versus the chosen design, -and important aspects to consider_: +4. _Alternative designs considered, and brief explanation of tradeoffs and decision versus the chosen design, and + important aspects to consider_: 1. Security. 2. Performance. 3. Maintainability. @@ -102,34 +96,24 @@ and important aspects to consider_: 1. _Conclusions/Decisions made in the design review sessions. Explanation why it’s selected_ ## Development Milestones -_This section described milestones for rather large projects so that we can set up intermediate goals for -better progress management. -For small projects, this may not be necessary._ + +_This section described milestones for rather large projects so that we can set up intermediate goals for better +progress management. For small projects, this may not be necessary._ ## Test Plan -_Describe in few sentences how the functionality will be tested. - How will we know that the implementation works as expected? How will we know nothing broke?_ +_Describe in few sentences how the functionality will be tested. How will we know that the implementation works as +expected? How will we know nothing broke?_ 1. _What unit tests would be added to cover the critical logic?_ 2. _What integration tests would be added to cover major components, if they cannot be tested by unit tests?_ 3. _Any tests to cover performance or Security concerns._ - -## References +## References _List any references here._ ## Appendix -_Add supporting information here, such as the results of performance studies and back-of-the-envelope calculations. +_Add supporting information here, such as the results of performance studies and back-of-the-envelope calculations. Organize it however you want but make sure it is comprehensible to your readers._ - - - - - - - - - diff --git a/docs/contributing/security.md b/docs/contributing/security.md new file mode 100644 index 00000000000..2c8b731aa2b --- /dev/null +++ b/docs/contributing/security.md @@ -0,0 +1,7 @@ +# Responsible Disclosure of Security Vulnerabilities + +Please do not file reports on GitHub for security issues. Please review the guidelines for reporting +[Security Vulnerabilities](https://www.linkedin.com/help/linkedin/answer/62924/security-vulnerabilities?lang=en). + +Alternatively, reports can be encrypted using PGP ([public key](https://www.linkedin.com/help/linkedin/answer/79676)) +and sent to security@linkedin.com, preferably with the title "GitHub linkedin/venice - <short summary>". diff --git a/docs/dev_guide/dev_guide.md b/docs/dev_guide/dev_guide.md deleted file mode 100644 index cceec2a5243..00000000000 --- a/docs/dev_guide/dev_guide.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -layout: default -title: Developer Guides -has_children: true -permalink: /docs/dev_guide ---- -# Developer Guide - -This section includes guides for Venice developers. - -New contributors should visit the [How to Contribute to Venice](./how_to/how_to.md) subsection to get started. - -Those looking for a general overview of the project structure may be interested to look at [Navigating the Project](navigating_project.md). - -The rest of this section is to document implementation details. \ No newline at end of file diff --git a/docs/dev_guide/how_to/CODE_OF_CONDUCT.md b/docs/dev_guide/how_to/CODE_OF_CONDUCT.md deleted file mode 100644 index 188be7ada30..00000000000 --- a/docs/dev_guide/how_to/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -layout: default -title: Code Of Conduct -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/code_of_conduct ---- - -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, caste, color, religion, or sexual -identity and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our -community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the overall - community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or advances of - any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email address, - without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement over -[Slack](https://venicedb.slack.com/archives/C03SLQWRSLF). -All complaints will be reviewed and investigated promptly and fairly. - -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series of -actions. - -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or permanent -ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within the -community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.1, available at -[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. - -Community Impact Guidelines were inspired by -[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. - -For answers to common questions about this code of conduct, see the FAQ at -[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at -[https://www.contributor-covenant.org/translations][translations]. - -[homepage]: https://www.contributor-covenant.org -[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html -[Mozilla CoC]: https://github.com/mozilla/diversity -[FAQ]: https://www.contributor-covenant.org/faq -[translations]: https://www.contributor-covenant.org/translations diff --git a/docs/dev_guide/how_to/CONTRIBUTING.md b/docs/dev_guide/how_to/CONTRIBUTING.md deleted file mode 100644 index 4323df9d256..00000000000 --- a/docs/dev_guide/how_to/CONTRIBUTING.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -layout: default -title: Contribution Agreement -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/contribution_agreement ---- - -# Contribution Agreement - -As a contributor, you represent that the code you submit is your -original work or that of your employer (in which case you represent -you have the right to bind your employer). By submitting code, you -(and, if applicable, your employer) are licensing the submitted code -to LinkedIn and the open source community subject to the BSD 2-Clause -license. - -# Responsible Disclosure of Security Vulnerabilities - -Please refer to our [Security Policy](./SECURITY.md) for our policy on -responsibly reporting security vulnerabilities. - -# Contribution Process - -The Venice community welcome everyone, and encourage a friendly and positive environment. - -**Contributions of various forms are welcome!** - -Please read existing GitHub issues or development work that is in progress or in the backlog to avoid duplication. If you are interested in those existing ones, you can leave a comment in the GitHub issues and the community will try to involve you. If you are not sure if it's duplicated, just create a GitHub issue and ask! - -For any PR, a GitHub issue is required. - -If you want to contribute something new, and it's not tracked in existing GitHub issues, please create a new -GitHub issue and the community will help review the idea. Please state `why` in your GitHub issue. -If you already have a short design in mind or change is not small please check the -[Venice Improvement Proposal](./design_doc.md) - -If you have any feature request, please create a new GitHub issue and the community will collect your feature request and work on it. - -If you have any user feedback, please create a new GitHub issue and the community will collect your feedback and work on it. - -## For New Contributors - -Please follow [Venice Workspace Setup](./workspace_setup.md) and -[Venice Recommended Development Workflow](./recommended_development_workflow.md) - -## Tips for Getting Your Pull Request Accepted - -1. Make sure all new features are tested and the tests pass. -2. Bug fixes must include a test case demonstrating the error that it - fixes. -3. Open an issue first and seek advice for your change before - submitting a pull request. Large features which have never been - discussed are unlikely to be accepted. - -## Contributor Sync Meeting - -Venice does Contributor sync meeting for the contributors to come togather in a virtual meeting. - -1. Meeting Frequency : Bi-weekly on Wednesday at 8 AM Pacific, 11 AM Eastern, 5 PM Central Europe -2. Meeting Co-ordination : Please join the #contributor-sync channel in the - [public Venice Slack](http://slack.venicedb.org/) -3. Meeting Length : One hour -4. Agenda : - 1. Discuss Venice Improvement Proposals. - 2. Review and address concerns for any open pull requests. - 3. Open discussion. - -## Code of Conduct - -This project and everyone who participates in it is governed by the [Venice Code of Conduct](./CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior on [Slack](https://venicedb.slack.com/archives/C03SLQWRSLF). \ No newline at end of file diff --git a/docs/dev_guide/how_to/SECURITY.md b/docs/dev_guide/how_to/SECURITY.md deleted file mode 100644 index 8433f1f1016..00000000000 --- a/docs/dev_guide/how_to/SECURITY.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -layout: default -title: Responsible Disclosure of Security Vulnerabilities -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/security_vulnerability ---- - -# Responsible Disclosure of Security Vulnerabilities - -Please do not file reports on GitHub for security issues. Please -review the guidelines for reporting [Security Vulnerabilities](https://www.linkedin.com/help/linkedin/answer/62924/security-vulnerabilities?lang=en). - -Alternatively, reports can be encrypted using PGP ([public key](https://www.linkedin.com/help/linkedin/answer/79676)) -and sent to security@linkedin.com, preferably with the title "GitHub -linkedin/venice - <short summary>". diff --git a/docs/dev_guide/how_to/design_doc.md b/docs/dev_guide/how_to/design_doc.md deleted file mode 100644 index 719070778d8..00000000000 --- a/docs/dev_guide/how_to/design_doc.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -layout: default -title: Design Documents -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/design_doc ---- -# Design Documents - -## Venice Improvement Proposal (VIP) - -Venice follow the Venice Improvement Proposal (VIP). VIP is the mechanism used to propose changes to the -Venice Codebase. - -Not all the changes require a VIP treatment, Please use your commonsense to check if writing this VIP help yourself -and the reviewers time instead of just doing it in the pull request itself. Generally, this applies to -* Large code refactors -* New functionality -* Public API changes - -In practical terms, the VIP defines a process in which developers can submit a design doc, -receive feedback and get the "go ahead" to execute. - -## Who can create a VIP? - -Any person willing to contribute to the Venice project is welcome to -create a VIP. - -## How does the VIP process work? - -A VIP proposal can be in these states: -1. **DRAFT**: (Optional) This might be used for contributors to collaborate and - to seek feedback on an incomplete version of the proposal. - -2. **DISCUSSION**: The proposal has been submitted to the community for - feedback and approval. - -3. **ACCEPTED**: The proposal has been accepted by the Venice project. - -4. **REJECTED**: The proposal has not been accepted by the Venice project. - -5. **IMPLEMENTED**: The implementation of the proposed changes have been - completed and everything has been merged. - -5. **RELEASED**: The proposed changes have been included in an official - Venice release. - - -The process works in the following way: - -1. The author(s) of the proposal will create a file named "VIP-xxx.md" in [proposal](../../proposals) folder - cloning from the [template for VIP proposals](../../proposals/VIP_TEMPLATE.md). The "xxx" number should - be chosen to be the next number from the existing VIP issues, listed [here](../../proposals/proposals.md) -2. The author(s) submit this file as a PR named "VIP-xxx: {short description}" in **DRAFT**/**DISCUSSION** stage. -3. People discuss using PR comments, each is its own threaded comment. - General comments can be made as general comment in the PR. There are two other ways for an interactive - discussion. - 1. Venice Community [Slack Channel](http://slack.venicedb.org/), Create a slack channel with #VIP-xxx - 2. Venice Contributor Sync Meeting, see details [here](./CONTRIBUTING.md) at Contributor Sync Meeting -4. Depending on the outcome of the discussion, the status could move to **ACCEPTED** or **REJECTED**, or it could stay - in **DISCUSSION** stage (e.g. if we agree tentatively on the broad strokes, but there are still action items to - refine certain aspects). At the end of this, the PR gets merged, and at that point the VIP will appear in the - directory of all historical VIPs. -5. If the merged VIP is still in **DISCUSSION** stage, then further PRs will be submitted to address - the remaining action items. -6. If the VIP is **ACCEPTED**, then next stages are implementation, with eventually some code change PR also - carrying an alteration of the VIP page to move the status to IMPLEMENTED. -7. All Pull Requests for this functionality, should prefix this "VIP-XXX" number in the title for quick access. - -## Acknowledgements - -This guide is inspired from the -[Apache Pulsar project proposal](https://github.com/apache/pulsar/blob/master/wiki/proposals/VIP.md) plan. - - - - diff --git a/docs/dev_guide/how_to/doc_guide.md b/docs/dev_guide/how_to/doc_guide.md deleted file mode 100644 index c53abf0d347..00000000000 --- a/docs/dev_guide/how_to/doc_guide.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -layout: default -title: Documentation Guideline -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/documentation_guideline ---- - -# Documentation Guideline - -We prefer simplicity and currently use GitHub page to host Venice documentation. Those documentation will be built -automatically by GitHub pipelines in the `main` branch. - -## General - -It is strongly encouraged that any code change which affects the validity of information in the docs also include -updates to the docs, so that both are kept in sync atomically. - -Experimental functionalities and future plans are also worth documenting, though they must be clearly marked as such, so -that users and operators reading those docs can make informed decisions about the level of risk they are willing to take -on if trying out a given functionality. If the level of maturity of a given functionality is not called out, then it -implicitly means that the functionality is considered mature and its API is unlikely to change. Undocumented configs and -APIs may or may not be considered mature and stable, and if in doubt, it is appropriate to open an Issue to request that -it be explicitly documented. - -In general, it is recommended to get familiar with the docs before writing more docs, to try to keep the style and -structure coherent. That being said, even if unsure where some documentation belongs, do err on the side of including it -(anywhere), and reviewers may suggest placing it elsewhere. - -## Hierarchy - -In order for your docs to be rendered properly in the documentation hierarchy, Venice developers need to add a header -section at the top of each documentation. The `title` section will be what the end user sees in the sidebar, and -the `parent` section represents the parent page of the current page for linking purpose. The `permalink` section will be -the URL path where the page will be served. An example of the header is below: - -``` ---- -layout: default -title: Documentation Guideline -parent: Developer Guides -permalink: /docs/dev_guide/documentation_guideline ---- -``` - -A page in the middle of the hierarchy has both the `parent` and `has_children` attributes. For example: - -``` ---- -layout: default -title: Write APIs -parent: User Guides -has_children: true -permalink: /docs/user_guide/write_api ---- -``` - -For a deeply nested page, a `grand_parent` attribute is also required. For example: - -``` ---- -layout: default -title: Push Job -parent: Write APIs -grand_parent: User Guides -permalink: /docs/user_guide/write_api/push_job ---- -``` - -Note that for now, the doc supports at most 3 levels of nesting. - -For more information, consult [Just the Docs](https://just-the-docs.github.io/just-the-docs/docs/navigation-structure/). - -## Pictures and Diagrams - -It is encouraged to use diagrams within the documentation, but there are some guidelines to standardize the way it is -done, and to avoid certain anti-patterns. - -### Text-based Assets - -For text-based assets (which are preferred whenever feasible), we wish to check them into source control. This should -include both the displayable asset (e.g. in SVG format) and the source file from which the displayable asset was -generated (e.g. in XML format). This makes the docs self-contained, and enables contributors to edit the assets over -time. - -Diagrams conforming to these guidelines can be placed under the `/docs/assets/images` path of the repo, and then -embedded in the docs with a relative link like this: - -```markdown -![](../../assets/images/vip_3_read_path.drawio.svg) -``` - -The [draw.io](https://draw.io) service makes it easy to generate such assets. If using [PlantUML](https://plantuml.com/starting), -it's recommended to generate diagrams into svg format by following this [guide](https://plantuml.com/svg). - -### Binary Assets - -For binary assets (e.g. PNG, BMP, JPG, etc.), we do NOT wish to check them into source control. Instead, they should be -linked from an external source. This can be done in GitHub itself. Within the Pull Request that proposes the doc change, -the contributor can insert images in the PR's description or comments, and then take the URL GitHub generated for it. -Then the modified files included in the PR can be edited to link to that image, and the PR updated. Externally hosted -images can be embedded with an absolute link like this: - -```markdown -![](https://user-images.githubusercontent.com/1248632/195111861-518f81c4-f226-4942-b88a-a34337da79e3.png) -``` - -## Emojis - -Here's a link to all the emojis available in README files: [Emoji Cheat Sheet](https://github.com/ikatyang/emoji-cheat-sheet/blob/master/README.md). - -## Testing Doc Changes - -There are two ways to test doc changes, locally and on the public web. Local testing is convenient to iterate quickly, -while public web testing is useful to make sure that nothing breaks (e.g., especially if changing styles, Ruby -dependencies, or Jekyll configs) and to share more significant documentation changes with PR reviewers. - -### Testing Locally - -The docs are rendered and served by a Ruby server called Jekyll. Follow the OS-specific instructions to [install Jekyll -and all its dependencies](https://jekyllrb.com/docs/installation/). After that, navigate to the `docs/` directory in the -repo, and run: - -```bash -bundle exec jekyll serve -``` - -Then navigate to `http://127.0.0.1:4000` and look at the docs in your browser. Whenever Markdown content or style files -change, the Jekyll server hot reloads them. In cases where the `Gemfile` or `_config.yml` files change, the server needs -to be restarted. Regarding the `Gemfile`, it is also important to note that not all gems are supported by GitHub Pages, -and so it is critical to perform the public web testing on the PR author's own GitHub fork (see next section). - -### Testing on the Public Web - -A GitHub fork can have its own documentation. This can be setup by: - -1. Navigating to the fork's Settings > Pages, i.e.: `https://github.com//venice/settings/pages` -2. Selecting which branch to publish the docs from. -3. Selecting `/docs` as the root directory. -4. Clicking Save. -5. Navigating to your fork's docs at: `https://.github.io/venice` - -For significant doc changes, please follow this process and add a link inside the PR to the docs hosted in the PR -author's own fork. \ No newline at end of file diff --git a/docs/dev_guide/how_to/how_to.md b/docs/dev_guide/how_to/how_to.md deleted file mode 100644 index 866ff6ccdf5..00000000000 --- a/docs/dev_guide/how_to/how_to.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: default -title: How to Contribute to Venice -has_children: true -parent: Developer Guides -permalink: /docs/dev_guide/how_to ---- -# How to Contribute to Venice - -This folder includes guides on how to set up the developer's environment, how to contribute designs, code, tests, docs, -diagrams and any other information relevant to contributors. \ No newline at end of file diff --git a/docs/dev_guide/how_to/recommended_development_workflow.md b/docs/dev_guide/how_to/recommended_development_workflow.md deleted file mode 100644 index 3fd6db85884..00000000000 --- a/docs/dev_guide/how_to/recommended_development_workflow.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -layout: default -title: Venice Recommended Development Workflow -parent: How to Contribute to Venice -grand_parent: Developer Guides -permalink: /docs/dev_guide/how_to/recommended_development_workflow ---- - -# Venice Recommended Development Workflow - -## Create a Design Document - -If your change is relatively minor, you can skip this step. If you are adding new major functionality, we suggest that -you add a design document and solicit comments from the community before submitting any code. - -Please follow the [Design Document Guide](./design_doc.md). - - -## Creating GitHub issue - -Every PR should be preceded by a GitHub issue to explain the problem statement unless it's a trivial bug fixes or a -documentation change. If your change is significant, please make sure your PR reviewers can align on the problem -statement via GitHub issues first. - -The GitHub issue should contain the detailed problem statement. - -## Pull Request -1. Fork the GitHub repository at http://github.com/linkedin/venice if you haven't already -2. Clone your fork, create a new branch, push commits to the branch -3. Consider whether documentation or tests need to be added or updated as part of the change, and add them as needed (doc changes should be submitted along with code change in the same PR) -4. Run all tests as described in the project's [Workspace setup guide](workspace_setup.md#run-the-test-suite). -5. Open a pull request against the `main` branch of `linkedin/venice`. (Only in special cases would the PR be opened against other branches.) -6. The PR title should usually be of the form `[component1]...[componentN]: Concise commit message`. - * Valid tags are: `[da-vinci]` (or `[dvc]`), `[server]`, `[controller]`, `[router]`, `[samza]`, - `[vpj]`, `[fast-client]` (or `[fc]`), `[thin-client]` (or `[tc]`), `[changelog]` (or `[cc]`), - `[producer]`, `[admin-tool]`, `[test]`, `[build]`, `[doc]`, `[script]`, `[compat]`, `[protocol]` - * `[compat]` tag means there are compatibility related changes in this PR, including upgrading protocol version, upgrading system store value schemas, etc. When there is a compatibility related change, it usually requires a specific deployment order, like upgrading controller before upgrading server. In this case, please explicitly call out the required deployment order in the commit message. -7. If the pull request is still a work in progress, and so is not ready to be merged, but needs to be pushed to GitHub to facilitate review, - then create the PR as a [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) PR - * If the PR cannot be created as a [draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) PR, - add `[WIP]` before the list of components. -8. Please state that the contribution is your original work and that you license the work to the project under the project's open source license. -9. If this PR resolves an issue be sure to include `Resolves #XXX` to correctly link and close the issue upon merge. -10. The project uses Apache Jenkins for continuous testing on Linux AMD64 and ARM64 build nodes. A CI job will not be started automatically for pull request. A maintainer has to trigger the testing. Feel free to tag a maintainer and ask for a build trigger. -11. Once ready, a maintainer will update the PR with the test results. -12. Investigate and fix failures caused by the pull the request -13. Fixes can simply be pushed to the same branch from which you opened your pull request. -14. Please address feedback via additional commits instead of amending existing commits. This makes it easier for the reviewers to know what has changed since the last review. All commits will be squashed into a single one by the committer via GitHub's squash button or by a script as part of the merge process. -15. Jenkins will automatically re-test when new commits are pushed. -16. Despite our efforts, Venice may have flaky tests at any given point, which may cause a build to fail. You need to ping committers to trigger a new build. If the failure is unrelated to your pull request and you have been able to run the tests locally successfully, please mention it in the pull request. - -## PR Description - -Describe - -* What changes to make and why you are making these changes. -* How are you going to achieve your goal -* Describe what testings you have done, for example, performance testing etc. - -Checklist that might be helpful to facilitate the review: - -* Design one-pager, design doc, or RFC -* GitHub Issue - -### Added new dependencies? - -Please list the new dependencies in the PR description and answer these questions for each new dependency - -* What's their license? -* Are they compatible with our license? - -## The Review Process -1. Other reviewers, including committers, may comment on the changes and suggest modifications. Changes can be added by simply pushing more commits to the same branch. -2. Please add a comment and "@" the reviewer in the PR if you have addressed reviewers' comments. Even though GitHub sends notifications when new commits are pushed, it is helpful to know that the PR is ready for review once again. -3. Lively, polite, rapid technical debate is encouraged from everyone in the community. The outcome may be a rejection of the entire change. -4. Reviewers can indicate that a change looks suitable for merging by approving it via GitHub's review interface. This indicates the strongest level of technical sign-off on a patch and it means: "I've looked at this thoroughly and take as much ownership as if I wrote the patch myself". If you approve a pull request, you will be expected to help with bugs or follow-up issues on the patch. Consistent, judicious use of pull request approvals is a great way to gain credibility as a reviewer with the broader community. Venice reviewers will typically include the acronym LGTM in their approval comment. This was the convention used to approve pull requests before the "approve" functionality was introduced by GitHub. -5. Sometimes, other changes will be merged which conflict with your pull request's changes. The PR can't be merged until the conflict is resolved. This can be resolved with `"git fetch upstream"` followed by `"git rebase upstream/main"` and resolving the conflicts by hand, then pushing the result to your branch. -6. Try to be responsive to the discussion rather than let days pass between replies. - -## Closing Your Pull Request / Issue -1. If a change is accepted, it will be merged and the pull request will automatically be closed, along with the associated Issue if the PR description contains `Resolves #XXX` -2. Note that in the rare case you are asked to open a pull request against a branch besides `main`, that you will actually have to close the pull request manually -3. If your pull request is ultimately rejected, please close it. -4. If a pull request has gotten little or no attention, consider improving the description or the change itself and ping likely reviewers after a few days. Consider proposing a change that's easier to include, like a smaller and/or less invasive change. -5. If a pull request is closed because it is deemed not the right approach to resolve an Issue, then leave the Issue open. However, if the review makes it clear that the problem identified in the Issue is not going to be resolved by any pull request (not a problem, won't fix) then also resolve the Issue. - -## Attribution & Acknowledgements - -This guide is based on the [Contributing Code Changes](https://cwiki.apache.org/confluence/display/KAFKA/Contributing+Code+Changes) guide from the [Apache Kafka](https://kafka.apache.org/) project. \ No newline at end of file diff --git a/docs/dev_guide/navigating_project.md b/docs/dev_guide/navigating_project.md deleted file mode 100644 index 31ac28cc752..00000000000 --- a/docs/dev_guide/navigating_project.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -layout: default -title: Navigating the Project -parent: Developer Guides -permalink: /docs/dev_guide/navigating_project ---- -# Navigating the Project - -The Venice codebase is split across these directories: - -- `clients`, which contains the user-facing libraries that most Venice users might be interested in. Those include: - - `da-vinci-client`, which is the stateful client, providing "eager caching" for the Venice datasets. [Learn more](../user_guide/read_api/da_vinci_client.md). - - `venice-admin-tool`, which is the shell tool intended for Venice operators. - - `venice-client`, which is a one-stop-shop for all clients which an online application might need, including - thin-client, fast-client, da-vinci-client, consumer. - - `venice-producer`, which enables an application to perform real-time writes to Venice. - - `venice-push-job`, which enables an offline job to push batch data to Venice. - - `venice-thin-client`, which is the most minimal dependency one can get to issue remote reads to the Venice backend, - by delegating as much of the query logic as possible to the Venice router tier. -- `integrations`, which contains additional libraries that some Venice users might be interested in, to connect Venice - with other third-party systems. The rule of thumb for including a module in this directory is that it should have - minimal Venice-specific logic, and be mostly just glue code to satisfy the contracts expected by the third-party - system. Also, these modules are intended to minimize the dependency burden of the other client libraries. Those - include: - - `venice-beam`, which implements the Beam Read API, enabling a Beam job to consume the Venice changelog. - - `venice-pulsar`, which contains an implementation of a Pulsar [Sink](https://pulsar.apache.org/docs/next/io-overview/#sink), - in order to feed data from Pulsar topics to Venice. - - `venice-samza`, which contains an implementation of a Samza [SystemProducer](https://samza.apache.org/learn/documentation/latest/api/javadocs/org/apache/samza/system/SystemProducer.html), - in order to let Samza stream processing jobs emit writes to Venice. -- `internal`, which contains libraries not intended for public consumption. Those include: - - `alpini`, which is a Netty-based framework used by the router service. It was forked from some code used by - LinkedIn's proprietary [Espresso](https://engineering.linkedin.com/espresso/introducing-espresso-linkedins-hot-new-distributed-document-store) - document store. At this time, Venice is the only user of this library, so there should be no concern of breaking - compatibility with other dependents. - - `venice-client-common`, which is a minimal set of APIs and utilities which the thin-client and other modules need - to depend on. This module used to be named `venice-schema-common`, as one can see if digging into the git history. - - `venice-common`, which is a larger set of APIs and utilities used by most other modules, except the thin-client. -- `services`, which contains the deployable components of Venice. Those include: - - `venice-controller`, which acts as the control plane for Venice. Dataset creation, deletion and configuration, - schema evolution, dataset to cluster assignment, and other such tasks, are all handled by the controller. - - `venice-router`, which is the stateless tier responsible for routing thin-client queries to the correct server - instance. It can also field certain read-only metadata queries such as cluster discovery and schema retrieval to - take pressure away from the controller. - - `venice-server`, which is the stateful tier responsible for hosting data, serving requests from both routers and - fast-client library users, executing write operations and performing cross-region replication. -- `tests`, which contains some modules exclusively used for testing. Note that unit tests do not belong here, and those - are instead located into each of the other modules above. - -Besides code, the repository also contains: - -- `all-modules`, which is merely an implementation detail of the Venice build and release pipeline. No code is expected - to go here. -- `docker`, which contains our various docker files. -- `docs`, which contains the wiki you are currently reading. -- `gradle`, which contains various hooks and plugins used by our build system. -- `scripts`, which contains a few simple operational scripts that have not yet been folded into the venice-admin-tool. -- `specs`, which contains formal specifications, in TLA+ and FizzBee, for some aspects of the Venice architecture. - -If you have any questions about where some contribution belongs, do not hesitate to reach out on the [community slack](http://slack.venicedb.org)! \ No newline at end of file diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt new file mode 100644 index 00000000000..4fd0c492195 --- /dev/null +++ b/docs/doc-requirements.txt @@ -0,0 +1,5 @@ +mkdocs +mkdocs-material +mkdocs-git-revision-date-localized-plugin +mkdocs-minify-plugin +mdx_truly_sane_lists \ No newline at end of file diff --git a/docs/quickstart/quickstart-multi-datacenter.md b/docs/getting-started/deployments/quickstart-multi-dc.md similarity index 81% rename from docs/quickstart/quickstart-multi-datacenter.md rename to docs/getting-started/deployments/quickstart-multi-dc.md index cff057de15e..09961efd3a0 100644 --- a/docs/quickstart/quickstart-multi-datacenter.md +++ b/docs/getting-started/deployments/quickstart-multi-dc.md @@ -1,41 +1,32 @@ ---- -layout: default -title: Venice Multi-Datacenter Docker Quickstart -parent: Quickstart -permalink: /docs/quickstart/quickstart-multi-datacenter ---- - - # Venice Multi-Datacenter Docker Quickstart - -Follow this guide to set up a multi-datacenter venice cluster using docker images -provided by Venice team. - +Follow this guide to set up a multi-datacenter venice cluster using docker images provided by Venice team. #### Step 1: Install and set up Docker Engine and docker-compose - Follow https://docs.docker.com/engine/install/ to install docker and start docker engine - +Follow [https://docs.docker.com/engine/install/](https://docs.docker.com/engine/install/) to install docker and start +docker engine #### Step 2: Download docker-compose-multi-dc-setup.yaml file + ``` wget https://raw.githubusercontent.com/linkedin/venice/main/docker/docker-compose-multi-dc-setup.yaml ``` - #### Step 3: Run docker compose to bring up Venice multi-colo setup + ```bash docker-compose -f docker-compose-multi-dc-setup.yaml up -d ``` - #### Step 4: Access `venice-client` container's bash shell + ```bash docker exec -it venice-client /bin/bash ``` #### Step 5: Make sure that you're in /opt/venice directory + ```bash # pwd /opt/venice @@ -43,27 +34,32 @@ docker exec -it venice-client /bin/bash cd /opt/venice ``` -#### Step 6: Create a store -Note: If you change the store name from `test-store` to something else, you will have to modify `/opt/venice/sample-data/multi-dc-configs/batch-push-job.properties` and `/opt/venice/sample-data/multi-dc-configs/inc-push-job.properties` to use the provided store name. +#### Step 6: Create a store + +Note: If you change the store name from `test-store` to something else, you will have to modify +`/opt/venice/sample-data/multi-dc-configs/batch-push-job.properties` and +`/opt/venice/sample-data/multi-dc-configs/inc-push-job.properties` to use the provided store name. ```bash -./create-store.sh http://venice-controller.dc-parent.venicedb.io:5555 venice-cluster0 test-store sample-data/schema/keySchema.avsc sample-data/schema/valueSchema.avsc +./create-store.sh http://venice-controller.dc-parent.venicedb.io:5555 venice-cluster0 test-store sample-data/schema/keySchema.avsc sample-data/schema/valueSchema.avsc ``` #### Step 7: Let's add a dataset to the store using batch push ##### Print dataset + ```bash -./avro-to-json.sh sample-data/batch-push-data/kv_records.avro +./avro-to-json.sh sample-data/batch-push-data/kv_records.avro ``` ##### Run a push job ```bash -./run-vpj.sh sample-data/multi-dc-configs/batch-push-job.properties +./run-vpj.sh sample-data/multi-dc-configs/batch-push-job.properties ``` ##### Fetch data from dc-0 + ```bash ./fetch.sh http://venice-router.dc-0.venicedb.io:7777 test-store 90 # should return a value ./fetch.sh http://venice-router.dc-0.venicedb.io:7777 test-store 100 # should return a value @@ -72,6 +68,7 @@ Note: If you change the store name from `test-store` to something else, you will ``` ##### Fetch data from dc-1 + ```bash ./fetch.sh http://venice-router.dc-1.venicedb.io:7777 test-store 90 # should return a value ./fetch.sh http://venice-router.dc-1.venicedb.io:7777 test-store 100 # should return a value @@ -79,20 +76,22 @@ Note: If you change the store name from `test-store` to something else, you will ./fetch.sh http://venice-router.dc-1.venicedb.io:7777 test-store 120 # should return null ``` - #### Step 8: Let's update some existing records in the dataset and add few new records using incremental push ##### Print records to be updated and added to the existing dataset in the store + ```bash -./avro-to-json.sh sample-data/inc-push-data/kv_records_v1.avro +./avro-to-json.sh sample-data/inc-push-data/kv_records_v1.avro ``` ##### Run incremental push job + ```bash -./run-vpj.sh sample-data/multi-dc-configs/inc-push-job.properties +./run-vpj.sh sample-data/multi-dc-configs/inc-push-job.properties ``` ##### Fetch data from dc-0 + ```bash ./fetch.sh http://venice-router.dc-0.venicedb.io:7777 test-store 90 # should return an unchanged value ./fetch.sh http://venice-router.dc-0.venicedb.io:7777 test-store 100 # should return an updated value @@ -101,6 +100,7 @@ Note: If you change the store name from `test-store` to something else, you will ``` ##### Fetch data from dc-1 + ```bash ./fetch.sh http://venice-router.dc-1.venicedb.io:7777 test-store 90 # should return an unchanged value ./fetch.sh http://venice-router.dc-1.venicedb.io:7777 test-store 100 # should return an updated value @@ -109,12 +109,14 @@ Note: If you change the store name from `test-store` to something else, you will ``` #### Step 9: Exit from the venice-client container + ```bash -# type exit command on the terminal or use cntrl + c +# type exit command on the terminal or use ctrl + c exit ``` #### Step 10: Teardown the cluster + ```bash docker-compose -f docker-compose-multi-dc-setup.yaml down ``` diff --git a/docs/quickstart/quickstart-single-datacenter.md b/docs/getting-started/deployments/quickstart-single-dc.md similarity index 67% rename from docs/quickstart/quickstart-single-datacenter.md rename to docs/getting-started/deployments/quickstart-single-dc.md index 55d71590fc8..200ac31fdf6 100644 --- a/docs/quickstart/quickstart-single-datacenter.md +++ b/docs/getting-started/deployments/quickstart-single-dc.md @@ -1,57 +1,55 @@ ---- -layout: default -title: Venice Single-Datacenter Docker Quickstart -parent: Quickstart -permalink: /docs/quickstart/quickstart-single-datacenter ---- - - # Venice Single-Datacenter Docker Quickstart - - -Follow this guide to set up a simple venice cluster using docker images -provided by Venice team. - +Follow this guide to set up a simple venice cluster using docker images provided by Venice team. #### Step 1: Install and set up Docker Engine and docker-compose - Follow https://docs.docker.com/engine/install/ to install docker and start docker engine +Follow [https://docs.docker.com/engine/install/](https://docs.docker.com/engine/install/) to install docker and start +docker engine #### Step 2: Download Venice quickstart Docker compose file + ``` wget https://raw.githubusercontent.com/linkedin/venice/main/docker/docker-compose-single-dc-setup.yaml ``` #### Step 3: Run docker compose -This will download and start containers for kafka, zookeeper, venice-controller, venice-router, -venice-server, and venice-client. -Once containers are up and running, it will create a test cluster, namely, `venice-cluster`. -Note: Make sure the `docker-compose-single-dc-setup.yaml` downloaded in step 2 is in the same directory from which you will run the following command. +This will download and start containers for kafka, zookeeper, venice-controller, venice-router, venice-server, and +venice-client. Once containers are up and running, it will create a test cluster, namely, `venice-cluster`. + +Note: Make sure the `docker-compose-single-dc-setup.yaml` downloaded in step 2 is in the same directory from which you +will run the following command. + ``` docker-compose -f docker-compose-single-dc-setup.yaml up -d ``` #### Step 4: Access `venice-client` container's bash shell -From this container, we will create a store in `venice-cluster`, which was created in step 3, push -data to it and run queries against it. + +From this container, we will create a store in `venice-cluster`, which was created in step 3, push data to it and run +queries against it. + ``` docker exec -it venice-client bash ``` #### Step 5: Create a venice store -The below script uses `venice-admin-tool` to create a new store: `venice-store`. -We will use the following key and value schema for store creation. + +The below script uses `venice-admin-tool` to create a new store: `venice-store`. We will use the following key and value +schema for store creation. key schema: + ```bash { "name": "key", "type": "string" } ``` + value schema: + ```bash { "name": "value", @@ -60,38 +58,48 @@ value schema: ``` Let's create a venice store: + ```bash ./create-store.sh http://venice-controller:5555 venice-cluster0 test-store sample-data/schema/keySchema.avsc sample-data/schema/valueSchema.avsc ``` #### Step 6: Push data to the store -Venice supports multiple ways to write data to the store. For more details, please refer to [Write Path](../README.md#write-path) section in [README](../README.md). -In this example, we will use batch push mode and push 100 records. + +Venice supports multiple ways to write data to the store. For more details, please refer to +[Write APIs](../../user-guide/write-apis/index.md). In this example, we will use batch push mode and push 100 records. + ``` key: 1 to 100 value: val1 to val100 ``` ##### Print dataset + ```bash -./avro-to-json.sh sample-data/batch-push-data/kv_records.avro +./avro-to-json.sh sample-data/batch-push-data/kv_records.avro ``` ##### Run a push job Let's push the data: + ```bash ./run-vpj.sh sample-data/single-dc-configs/batch-push-job.properties ``` #### Step 7: Read data from the store -Let's query some data from `venice-store`, using `venice-thin-client` which is included in venice container images. (key: `1 to 100`) + +Let's query some data from `venice-store`, using `venice-thin-client` which is included in venice container images. +(key: `1 to 100`) + ``` ./fetch.sh ``` + For example: + ```bash -$ ./fetch.sh http://venice-router:7777 test-store 1 +$ ./fetch.sh http://venice-router:7777 test-store 1 key=1 value=val1 @@ -105,26 +113,29 @@ key=101 value=null ``` - #### Step 8: Update and add some new records using Incremental Push -Venice supports incremental push which allows us to update values of existing rows or to add new rows in an existing store. -In this example, we will + +Venice supports incremental push which allows us to update values of existing rows or to add new rows in an existing +store. In this example, we will + 1. update values for keys from `91-100`. For example, the new value of `100` will be `val100_v1` 2. add new rows (key: `101-110`) ##### Print records to be updated and added to the existing dataset in the store + ```bash -./avro-to-json.sh sample-data/inc-push-data/kv_records_v1.avro +./avro-to-json.sh sample-data/inc-push-data/kv_records_v1.avro ``` ##### Run incremental push job + ```bash ./run-vpj.sh sample-data/single-dc-configs/inc-push-job.properties ``` #### Step 9: Read data from the store after Incremental Push -Incremental Push updated the values of keys 91-100 and added new rows 101-110. -Let's read the data once again. + +Incremental Push updated the values of keys 91-100 and added new rows 101-110. Let's read the data once again. ```bash # Value of 1 changed remains unchanged @@ -143,20 +154,24 @@ key=101 value=val101 ``` - #### Step 10: Exit `venice-client` + ```bash # type exit command on the terminal or use ctrl + c exit ``` #### Step 11: Stop docker + Tear down the venice cluster + ```bash docker-compose -f docker-compose-single-dc-setup.yaml down ``` ## Next steps -Venice is a feature rich derived data store. It offers features such as write-compute, read-compute, streaming ingestion, multi data center active-active replication, -deterministic conflict resolution, etc. To know more about such features please refer [README](../README.md) and reach out to -the [Venice team](../README.md#community-resources). + +Venice is a feature rich derived data store. It offers features such as write-compute, read-compute, streaming +ingestion, multi data center active-active replication, deterministic conflict resolution, etc. To know more about such +features please refer to the [User Guide](../../user-guide/index.md) and reach out to the +[Venice team](../../README.md#community). diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md new file mode 100644 index 00000000000..c238036b7af --- /dev/null +++ b/docs/getting-started/index.md @@ -0,0 +1,14 @@ +# Getting Started + +**New to Venice?** Follow this path to get up and running. + +## Learn Venice + +- [Architecture Overview](learn-venice/architecture-overview.md) - Core concepts +- [Merging Batch & Real-Time Data](learn-venice/merging-batch-and-rt-data.md) - Hybrid stores, lambda vs kappa + architecture + +## Deployments + +- [Single DC Quickstart](deployments/quickstart-single-dc.md) - Basic setup +- [Multi DC Quickstart](deployments/quickstart-multi-dc.md) - Active-active replication diff --git a/docs/getting-started/learn-venice/architecture-overview.md b/docs/getting-started/learn-venice/architecture-overview.md new file mode 100644 index 00000000000..9b442409efa --- /dev/null +++ b/docs/getting-started/learn-venice/architecture-overview.md @@ -0,0 +1,50 @@ +# Architecture Overview + +Venice is a derived data platform that bridges the offline, nearline, and online worlds. Before diving into the +quickstarts, understanding these core concepts will help you get the most out of Venice. + +**Prefer video?** Watch the [Open-Sourcing Venice conference talk](https://www.youtube.com/watch?v=pJeg4V3JgYo) for a +comprehensive overview of Venice's architecture, use cases, and real-world deployment at LinkedIn scale. + +## Core Concepts + +### Stores & Versions + +- A **store** is a dataset (like a table or collection) +- Each store consists of immutable **versions** that enable zero-downtime updates +- When you push new data, Venice creates a new version and atomically swaps read traffic + +### Partitions & Replication + +- Data is horizontally partitioned for scalability +- Each partition is replicated across multiple nodes for availability +- Supports **active-active replication** between regions with CRDT-based conflict resolution + +### Write Modes + +Venice supports three write granularities that can be mixed: + +- **Batch Push**: Full dataset replacement (from Hadoop, Spark, etc.) +- **Incremental Push**: Insert many rows into existing dataset +- **Streaming Writes**: Real-time updates via Kafka, Samza, or Online Producer +- **Hybrid Stores**: Combine batch + streaming for powerful Lambda/Kappa patterns + +### Read Modes + +Choose your performance/cost tradeoff: + +| Client Type | Network Hops | Latency (p99) | State Footprint | +| ------------------- | ------------ | --------------------------- | -------------------------- | +| **Thin Client** | 2 | < 10ms | Stateless | +| **Fast Client** | 1 | < 2ms | Minimal (routing metadata) | +| **Da Vinci Client** | 0 | < 1ms (SSD) or < 10μs (RAM) | Full dataset locally | + +All clients support the same APIs: single get, batch get, and read compute operations. + +## Key Features + +- **Active-Active Replication**: Write to multiple regions, automatic conflict resolution +- **Write Compute**: Partial updates and collection merging operations +- **Read Compute**: Server-side projections, vector operations (dot product, cosine similarity) +- **Time-to-Live (TTL)**: Automatic data expiration for compliance and storage efficiency +- **Multi-tenancy**: Share clusters across teams with resource isolation diff --git a/docs/user_guide/design_patterns/merging_batch_and_rt_data.md b/docs/getting-started/learn-venice/merging-batch-and-rt-data.md similarity index 77% rename from docs/user_guide/design_patterns/merging_batch_and_rt_data.md rename to docs/getting-started/learn-venice/merging-batch-and-rt-data.md index 3d14a2b48ee..ff85402aabb 100644 --- a/docs/user_guide/design_patterns/merging_batch_and_rt_data.md +++ b/docs/getting-started/learn-venice/merging-batch-and-rt-data.md @@ -1,11 +1,3 @@ ---- -layout: default -title: Merging Batch & Real-Time Data -parent: Design Patterns -grand_parent: User Guides -permalink: /docs/user_guide/design_patterns/merging_batch_and_rt_data ---- - # Merging Batch & Real-Time Data Venice being a derived data platform, an important category of use cases is to merge batch data sources and real-time @@ -33,8 +25,8 @@ the periphery are the two pipelines merged together, presumably within the user' ### Implementing the Lambda Architecture in Venice -The Lambda Architecture, exactly as it is proposed by Nathan Marz, can be implemented using two Venice stores: a -batch-only one, and a real-time-only one. The application can query both stores and implement whatever arbitrary +The Lambda Architecture, exactly as it is proposed by Nathan Marz, can be implemented using two Venice stores: a +batch-only one, and a real-time-only one. The application can query both stores and implement whatever arbitrary reconciliation logic they wish. ## Kappa Architecture @@ -42,28 +34,29 @@ reconciliation logic they wish. The Kappa Architecture was proposed by [Jay Kreps](https://github.com/jkreps) in 2014, in a blog post titled [Questioning the Lambda Architecture](https://www.oreilly.com/radar/questioning-the-lambda-architecture/). -In a nutshell, the idea is that the Lambda Architecture's two parallel pipelines are cumbersome to maintain, especially +In a nutshell, the idea is that the Lambda Architecture's two parallel pipelines are cumbersome to maintain, especially given that they would be implemented in different technologies (some batch processing framework and a stream processing one). The Kappa Architecture, on the other hand, proposes to use only a stream processor, but to configure it to run -either in "real-time processing" or in "historical reprocessing" mode. Whenever data needs to be reprocessed, a new -pipeline can be instantiated, configured to start from the beginning of the historical input, output all of its -processed data into a new instance of a database, and then keep going to process real-time events after that. The +either in "real-time processing" or in "historical reprocessing" mode. Whenever data needs to be reprocessed, a new +pipeline can be instantiated, configured to start from the beginning of the historical input, output all of its +processed data into a new instance of a database, and then keep going to process real-time events after that. The application can then switch over its read traffic to the new database instance. ![](https://dmgpayxepw99m.cloudfront.net/kappa-61d0afc292912b61ce62517fa2bd4309.png) -### Implementing the Lambda Architecture in Venice + +### Implementing the Kappa Architecture in Venice The Kappa Architecture, exactly as it is proposed by Jay Kreps, can again be implemented using two Venice stores, except -that both of them are real-time-only. The user does need to manually keep creating new stores and manually switch over +that both of them are real-time-only. The user does need to manually keep creating new stores and manually switch over the reads to the new one when ready. It does achieve the purpose of maintaining only a stream processing stack, and it -gives precise control over which version of the data to query, but it may be more tedious on an ongoing basis as +gives precise control over which version of the data to query, but it may be more tedious on an ongoing basis as compared to the Hybrid Store (see below). ## Hybrid Store -The Hybrid Store Design Pattern has been described both in [blog](https://philosopherping.com/hybrid-store-design-pattern) -and in video ([1](https://www.youtube.com/watch?v=hc0pgvnr3fQ), [2](https://www.youtube.com/watch?v=mM-6GysXii4)) -formats. +The Hybrid Store Design Pattern has been described both in +[blog](https://philosopherping.com/hybrid-store-design-pattern) and in video +([1](https://www.youtube.com/watch?v=hc0pgvnr3fQ), [2](https://www.youtube.com/watch?v=mM-6GysXii4)) formats. Venice supports this pattern out of the box, and it is likely the simplest way to merge batch and real-time data, as -Venice handles the whole orchestration on behalf of the user. \ No newline at end of file +Venice handles the whole orchestration on behalf of the user. diff --git a/docs/operations/advanced/data-integrity.md b/docs/operations/advanced/data-integrity.md new file mode 100644 index 00000000000..672c3771e3c --- /dev/null +++ b/docs/operations/advanced/data-integrity.md @@ -0,0 +1,143 @@ +# Venice Data Integrity Validation + +### What is Data Integrity Validation (DIV)? + +- Validate and detect errors in transmitted (Kafka) data. +- The sender (VeniceWriter) calculates and includes the DIV metadata. +- The receiver recalculates and verifies the DIV metadata on arrival. + +### What does DIV metadata look like? + +- Sender transmits data in segments. +- Sender decides when to close the current segment and start a new one. +- A segment contains multiple Kafka records + - Start of Segment (SOS) + - End of Segment (EOS) + - Control or data messages between +- GUID + - One unique ID per-sender (VeniceWriter). + - GUID changes if VeniceWriter or the server restarts. +- Sequence number + - Within a segment, the sequence number starts at 0. + +### Data integrity issues in Venice: + +- A segment starts from a non-zero sequence number (UNREGISTERED_PRODUCER). +- A gap exists between or within segments (MISSING). +- Data within a segment is corrupted (CORRUPT). +- Producers have produced duplicate messages, which is expected due to retries (DUPLICATE). + +### Different behaviors before and after End Of Push (EOP) + +- End Of Push (EOP) is a control message in the Kafka topic sent once per partition at bulk load end, after all data + producers come online. +- When pushing a new store version, if DIV detects fatal data validation issues (including UNREGISTERED_PRODUCER, + MISSING, and CORRUPT) while consuming batch push data before EOP, the ingestion task errors and aborts. +- If issues appear after receiving EOP, the ingestion task continues, but DIV logs a warning about data integrity + issues. + +### Data structures + +- Sender - VeniceWriter/Segment +- Data - ProducerRecord/KafkaMessageEnvelope/ProducerMetadata +- Receiver - ConsumptionTask/StoreIngestionTask + +### Persistence + +- DIV needs to be checkpointed for several reasons: + - DIV state would be lost on server restarts leading to potential data integrity issues undetected (inaccuracy) + - Otherwise, without checkpointing, DIV needs to rebuild its state from the beginning of the topic on every restart + (inefficiency) + - In case of a crash or restart, we need to know where to resume validation from (last checkpoint). + +### Why Two Separate DIV validators? + +- Two state pipeline +- Stage 1: Consumer Thread + + - Kafka → Consumer Thread → Validation → Queue → ... + - Reads messages from Kafka topics + - Performs validation using Consumer DIV (RT or remote VT) + - Queues messages to StoreBufferService + +- Stage 2: Drainer Thread + + - ... → Queue → Drainer Thread → Validation → Storage Engine + - Dequeues messages from StoreBufferService + - Performs validation using drainer DIV (all topics) + - Persists data to storage engine + - Checkpoints offsets to disk. + +- Consumer is always ahead of drainer + + - The consumer thread reads from Kafka faster than the drainer can persist to disk. There's buffering in between. + - Kafka producer buffers (for leaders) + - StoreBufferService queue etc. + - Time T1: Consumer validates message at offset 1000 → Consumer DIV state = offset 1000 + - Time T2: Consumer validates message at offset 2000 → Consumer DIV state = offset 2000 + - Time T3: Drainer persists message at offset 1000 → Drainer DIV state = offset 1000 + - If we only had one consumer DIV, we couldn't checkpoint the correct state to disk. + +- DIV State Must Match Persisted Data + + - The DIV checkpoint saved to disk must exactly match what's actually persisted in the storage engine. Since the + drainer is responsible for persistence, only the Drainer DIV state can be safely checkpointed. + +- Leader Double Validation + - Consumer DIV (in consumer thread): Validates RT messages before producing to VT + - Drainer DIV (in drainer thread): Validates same messages again before persisting (unnecessary) + +### DIV in State Transitions (per-partition) + +- **OFFLINE -> STANDBY:** DIV state restoration. The DIV state is restored from the persisted OffsetRecord (drainerDiv). + In STANDBY mode, the DIV continues to validate incoming messages against the restored state and keeps it updated. +- **STANDBY -> OFFLINE:** The DIV state is cleared immediately without being persisted to disk. Any DIV state + accumulated since the last checkpoint is lost The system relies on periodic checkpoints during consumption, not + on-demand checkpoints during state transitions. This design choice prioritizes fast un-subscription over preserving + the absolute latest DIV state. +- **STANDBY -> LEADER:** Wipes all DIV state for the partition in the consumer DIV. Copies the producer states from the + drainer's DIV validator to the consumer DIV. +- **LEADER -> STANDBY:** The drainer DIV maintains producer states that have been validated and persisted. If this + replica becomes leader again, it will clone the drainer DIV state. + +### Kafka Log Compaction impacts DIV in Venice + +- When Kafka log compaction runs, it: + - Deletes duplicate records (identified by the key) and keeps only the latest record. + - Creates **gaps** in sequence numbers that DIV would normally flag as data loss. + - Venice uses a **log compaction delay threshold** to distinguish between: + - **Real data loss** (missing messages within the compaction window) + - **Expected gaps** (missing messages beyond the compaction window) + - Example: + - T0: Producer sends messages with seq# 1, 2, 3, 4, 5 + - T1: Kafka stores all messages + - T2: min.compaction.lag.ms = 24 hours passes + - T3: Kafka compaction runs, deletes messages 2, 3 (duplicate keys) + - T4: Consumer reads: seq# 1, 4, 5 (gap detected!) + - DIV Check: + - Last message timestamp: T0 + - Current time: T4 (> 24 hours later) + - elapsedTime >= min.compaction.lag.ms? YES + - Result: TOLERATE the gap (expected compaction) + - T4: Consumer reads: seq# 1, 4, 5 (gap detected!) + - elapsedTime < min.compaction.lag.ms? YES + - Result: THROW EXCEPTION (data loss!) + +### DIV cleanup mechanism + +- Prevents DIV from accumulating unbounded producer state by automatically expiring and removing old producer tracking + information. +- Without cleanup, this state grows indefinitely + - Large memory footprint - State accumulates in heap + - Checkpoint overhead - All producer states are persisted to disk on every offset sync +- Solution: Time-Based Expiration + - **MaxAge** allows Venice to automatically expire old producer state that's no longer relevant. + - Only applied to drainer DIV (not consumer DIV, which is transient) + - For hybrid stores, maxAge is >= rewind time. + - Can be disabled by setting to -1. +- How it works: + - When checkpointing offsets, DIV clears expired state. + - When loading state from disk (e.g., after restart), maxAge is also applied. This prevents loading stale state after + a restart. + - Why MaxAge ≥ Rewind Time for Hybrid Stores? + - Support RT replay, otherwise it doesn't recognize producer’s div state in RT. diff --git a/docs/quickstart/quickstart_P2P_transfer_bootstrapping.md b/docs/operations/advanced/p2p-bootstrapping.md similarity index 92% rename from docs/quickstart/quickstart_P2P_transfer_bootstrapping.md rename to docs/operations/advanced/p2p-bootstrapping.md index b0857d6ef08..af692986929 100644 --- a/docs/quickstart/quickstart_P2P_transfer_bootstrapping.md +++ b/docs/operations/advanced/p2p-bootstrapping.md @@ -1,16 +1,8 @@ ---- -layout: default -title: Venice P2P Transfer Bootstrapping Quickstart -parent: Quickstart -permalink: /docs/quickstart/quickstart_P2P_transfer_bootstrapping ---- - - # Venice P2P Transfer Bootstrapping Architecture **High-Performance Peer-to-Peer Data Bootstrap** -*Accelerating Venice Node Deployment Through Direct Peer-to-Peer RocksDB Snapshot Transfer* +_Accelerating Venice Node Deployment Through Direct Peer-to-Peer RocksDB Snapshot Transfer_ --- @@ -19,11 +11,14 @@ permalink: /docs/quickstart/quickstart_P2P_transfer_bootstrapping ### The Problem **Bootstrap Bottlenecks** + - **Kafka-Only Recovery:** All nodes bootstrap exclusively from Kafka brokers -- **Resource Intensive:** Time-consuming process during cluster recovery, inefficient for consuming messages from the PubSub system under high-update workloads +- **Resource Intensive:** Time-consuming process during cluster recovery, inefficient for consuming messages from the + PubSub system under high-update workloads - **Scalability Limits:** Broker capacity becomes recovery bottleneck **Real-World Impact** + - Extended MTTR (Mean Time to Restore or Repair) during outages - Cascading broker overload - Slower cluster expansion @@ -32,16 +27,17 @@ permalink: /docs/quickstart/quickstart_P2P_transfer_bootstrapping ### 💡 The Solution **Direct P2P Transfer** + - **Peer-to-Peer:** Direct node-to-node data transfer - **RocksDB Snapshots:** Consistent point-in-time data copies - **Intelligent Fallback:** Automatic Kafka Ingestion recovery on failure - **Low Risk:** Low Deployment Risk in DaVinci client **Key Benefits** + - Faster node recovery and scaling - Reduced Kafka broker load - --- ## 🏗️ System Architecture Flow @@ -66,32 +62,32 @@ Step 1: Peer Discovery └─────────────┘ Step 2: Sequential Host Attempts -┌─────────────┐ T1:Try Host 1 ┌─────────────┐ -│ Client Node │ ──────────────> │ Server A │ -│ │ <── FAIL ────── │ (No Data/ │ -└─────────────┘ │ Busy) │ - └─────────────┘ - T2:Try Host 2 ┌─────────────┐ - ──────────────> │ Server B │ +┌─────────────┐ T1:Try Host 1 ┌─────────────┐ +│ Client Node │ ──────────────> │ Server A │ +│ │ <── FAIL ────── │ (No Data/ │ +└─────────────┘ │ Busy) │ + └─────────────┘ + T2:Try Host 2 ┌─────────────┐ + ──────────────> │ Server B │ <── FAIL ────── │ Table Format│ - │ not Match │ - └─────────────┘ - - T3:Try Host 3 ┌─────────────┐ - ──────────────> │ Server C │ - │ (Not busy, │ - │Format Match)│ - └─────────────┘ - -Step 3: Start Transfer -┌────────────────┐ -│ Client Node │ + │ not Match │ + └─────────────┘ + + T3:Try Host 3 ┌─────────────┐ + ──────────────> │ Server C │ + │ (Not busy, │ + │Format Match)│ + └─────────────┘ + +Step 3: Start Transfer +┌────────────────┐ +│ Client Node │ │ │ │ Receives: │ <── 1.1 File: file1.sst ────────── ┌─────────────┐ │ 1. Files │ <── 1.2 File: file2.sst ────────── │ Server C │ │ │ <── 1.3 File: MANIFEST_00 ──────── │ │ │ 2. Metadata │ <── 2. Metadata ────────────────── └─────────────┘ -│ 3. COMPLETE │ <── 3. COMPLETE Flag ───────────── +│ 3. COMPLETE │ <── 3. COMPLETE Flag ───────────── └────────────────┘ Step 4: Client Processing after recevie COMPLETE flag @@ -113,10 +109,12 @@ Step 5: Kafka Ingestion Fallback (Always Happens) ``` **Step 1: Peer Discovery** + - **Venice Server:** Query Helix CustomizedViewRepository for COMPLETED nodes - **DaVinci Application:** Query DaVinci push job report for ready-to-serve nodes **Step 2: P2P Transfer in Client Side** + ``` Connectivity Check ────> Peer Selection ────> Sequential Request ↓ ↓ ↓ @@ -125,15 +123,20 @@ check with caching connectable hosts success or exhaust ``` **Step 3: Data Transfer** + ``` Snapshot Creation ────> File Streaming ────> Metadata Sync ↓ ↓ ↓ Server creates Chunked transfer Offset + Version State RocksDB snapshot with MD5 validation after files complete ``` + **Step 4: Completion** -- **Success Path:** After validating all files, atomically rename the temp directory to the partition directory, then initiate Kafka ingestion to synchronize any remaining offset gap. -- **Fallback Path:** If any error occurs, clean up the temp directory and retry with the next peer; if all peers fail, back to Kafka bootstrap from the beginning. + +- **Success Path:** After validating all files, atomically rename the temp directory to the partition directory, then + initiate Kafka ingestion to synchronize any remaining offset gap. +- **Fallback Path:** If any error occurs, clean up the temp directory and retry with the next peer; if all peers fail, + back to Kafka bootstrap from the beginning. ### Key Components @@ -142,7 +145,6 @@ RocksDB snapshot with MD5 validation after files complete - **BlobSnapshotManager** - Server-side snapshot lifecycle - **NettyFileTransferClient** - High-performance client - ## 📥 Client-Side Process ### Process Flow @@ -154,23 +156,26 @@ RocksDB snapshot with MD5 validation after files complete ### 🔍 Step 1: Peer Discovery **Venice Server:** + - Query Helix CustomizedViewRepository - Find COMPLETED nodes - Filter by store/version/partition **DaVinci Application:** + - Query DaVinci push job report - Find ready-to-serve nodes - Extract available peer list ### 🔗 Step 2: Connectivity Checking + Smart Caching Strategy due to large peer sets: ``` ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ ┌─────────────┐ │ Purge Stale │ → │ Filter Bad │ → │ Check Hosts in │ → │ Update Cache│ │ Records │ │ Hosts │ │Parallel Connectivity│ │ Results │ -└─────────────┘ └─────────────┘ └─────────────────────┘ └─────────────┘ +└─────────────┘ └─────────────┘ └─────────────────────┘ └─────────────┘ ``` - **Parallel Testing:** Multiple host connections simultaneously @@ -187,22 +192,24 @@ For Each Peer (Shuffled List): └─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ ``` - ### 🚨 Error Handling Details in different Scenarios **Connection Failures (VenicePeersConnectionException)** + - Network timeout - SSL handshake failure - Host unreachable - **Action:** Try next peer **File Not Found (VeniceBlobTransferFileNotFoundException)** + - Snapshot missing - Stale snapshot - Server too busy, reject with 429 errors - **Action:** Try next peer **Transfer Errors (Data Integrity Issues)** + - Checksum mismatch - File size mismatch - Network interruption: Server initiates deployment or shuts down unexpectedly @@ -223,10 +230,12 @@ For Each Peer (Shuffled List): ### 🔄 Step 4: Kafka Fallback Strategy **Two-Phase Strategy:** + - **Phase 1 - Blob Transfer:** Rapid bulk data transfer via P2P - **Phase 2 - Kafka Fill-Gap:** Even if blob transfer fails, deployment continues with Kafka ingestion -**Zero-Risk Design:** After a successful blob transfer, Kafka ingestion always follows to synchronize any data between the snapshot offset and the latest offset, guaranteeing full deployment. +**Zero-Risk Design:** After a successful blob transfer, Kafka ingestion always follows to synchronize any data between +the snapshot offset and the latest offset, guaranteeing full deployment. --- @@ -241,6 +250,7 @@ For Each Peer (Shuffled List): ### ☑️ Step 1: Request Reception & Validation Incoming Request Pipeline: + ``` ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ HTTP GET │ │ Parse URI │ │ Validate │ │ Check │ @@ -256,12 +266,14 @@ Incoming Request Pipeline: ### 📊 Step 2: Prepare Snapshot & Metadata **Snapshot Lifecycle:** + - Check staleness (Configable TTL) - Verify concurrent user limits - Create a new snapshot if the existing one is stale or does not exist - Prepare partition metadata **Metadata Preparation:** + - Serialization of StoreVersionState (enables synchronization of hybrid store configuration parameters) - OffsetRecord encapsulation (captures the snapshot’s offset for accurate state synchronization) - JSON metadata response @@ -270,8 +282,6 @@ Incoming Request Pipeline: **Server-Side Adaptive Chunking Algorithm:** - - ``` Step 1: File Preparation ┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ @@ -283,13 +293,13 @@ Step 1: File Preparation └─────────────────────────┘ └─────────────────────────┘ └─────────────────────────┘ └─────────────────────────┘ └─────────────────────────┘ Step 2: Adaptive Chunking Strategy -┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ -│ Calculate Optimal │ │ Create Chunked │ │ Wrap for HTTP │ -│ Chunk Size │ │ File Handler │ │ Streaming │ -│ 16KB - 2MB │ → │ │ → │ │ -│ Determine best chunk │ │ Set up memory-efficient │ │ Prepare for HTTP │ -│ size based on file │ │ streaming mechanism │ │ chunked transfer │ -└─────────────────────────┘ └─────────────────────────┘ └─────────────────────────┘ +┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ +│ Calculate Optimal │ │ Create Chunked │ │ Wrap for HTTP │ +│ Chunk Size │ │ File Handler │ │ Streaming │ +│ 16KB - 2MB │ → │ │ → │ │ +│ Determine best chunk │ │ Set up memory-efficient │ │ Prepare for HTTP │ +│ size based on file │ │ streaming mechanism │ │ chunked transfer │ +└─────────────────────────┘ └─────────────────────────┘ └─────────────────────────┘ Step 3: Efficient File Streaming ┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ @@ -304,9 +314,6 @@ Step 3: Efficient File Streaming **Client-Side High-Performance Reception:** - - - ``` Step 1: File Setup ┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ @@ -337,6 +344,7 @@ Step 3: Complete File └─────────────────────────┘ └─────────────────────────┘ └─────────────────────────┘ ``` + ``` Data Flow Transformation at Convert to Channel @@ -353,6 +361,7 @@ Network → ByteBuf → ByteBufInputStream → ReadableByteChannel → FileChann ### 📊 Step 4: Metadata Transfer & Completion Protocol **Critical Ordering for Data Consistency:** + ``` ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ 1. Prepare │ │ 2. Transfer │ │ 3. Send │ │ 4. Send │ @@ -365,27 +374,32 @@ Network → ByteBuf → ByteBufInputStream → ReadableByteChannel → FileChann **Why This Ordering is Critical:** ❌ **Wrong Order (Metadata → Files → Complete):** + - Client updates offset records immediately - If file transfer fails, offset state is corrupted - Need offset/SVS rollback mechanisms - Increased error handling complexity and risk ✅ **Correct Order (Files → Metadata → Complete):** + - Files transferred and validated first - Metadata only sent after successful file transfer - Atomic state update on COMPLETE signal - Less risk consistency guarantee **Metadata Consistency Protocol:** + - **Metadata:** OffsetRecord + StoreVersionState captured before snapshot creation time - **JSON Serialization:** Structured metadata transfer with size validation --- + ## 🌐 Traffic Control ### 🚥 Global Traffic Shaping -**Shared Rate Limiting:** Single GlobalChannelTrafficShapingHandler instance controls bandwidth across ALL blob transfer channels globally +**Shared Rate Limiting:** Single GlobalChannelTrafficShapingHandler instance controls bandwidth across ALL blob transfer +channels globally ### Traffic Management Strategy @@ -405,6 +419,7 @@ Global Control Architecture: ### Adaptive Throttling System **Dynamic Rate Adjustment:** + - Increment/Decrement: 20% - Range: 20% - 200% of base rate - Separate read/write throttlers @@ -418,32 +433,38 @@ Global Control Architecture: 📥 **Receiver/Client Feature Enablement Control** -*Venice Server:* +_Venice Server:_ + - Store Level: `blobTransferInServerEnabled` - Application Level: `blob.transfer.receiver.server.policy` -*DaVinci Application:* +_DaVinci Application:_ + - Store Level: `blobTransferEnabled` 📤 **Sender/Server Feature Enablement Control** -*All Applications:* +_All Applications:_ + - Application Level: `blob.transfer.manager.enabled` ### Performance Tuning Parameters 🪜 **Thresholds** + - **Offset Lag:** Skip blob transfer if not lagged enough - **Snapshot TTL:** Maintain snapshot freshness - **Snapshot Cleanup Interval:** Maintain disk storage 🏎️ **Bandwidth Limits** + - **Client Read:** Client side read bytes per sec - **Service Write:** Server write bytes per sec - **Adaptive Range:** 20% - 200% of base rate - **Max Concurrent Snapshot User:** Control server concurrent serve requests load ⏰ **Timeouts** + - **Transfer Max:** Server side max timeout for transferring files - **Receive Max:** Client side max timeout for receiving files @@ -454,26 +475,31 @@ Global Control Architecture: ### Venice Blob Transfer: Key Features & Benefits **🚀 Performance Excellence** + - Intelligent peer discovery and selection - Fast failover with comprehensive error handling - High-performance Netty streaming architecture - Efficient file operations and adaptive chunking **🔒 Rock-Solid Reliability** + - Consistent RocksDB snapshots with point-in-time guarantees - Comprehensive error handling and automatic cleanup - Data integrity validation with MD5 and size checks - Automatic Kafka fallback for 100% data coverage **🛡️ Security & Control** + - End-to-end SSL/TLS encryption - Certificate-based ACL validation - Global traffic rate limiting and adaptive throttling - Comprehensive timeout and connection management **🔧 Operational Excellence** + - Flexible multi-level configuration - Automated snapshot lifecycle management - Graceful degradation and low risk deployment -**Result: Faster node recovery, reduced infrastructure load, improved cluster scalability with enterprise-grade reliability** \ No newline at end of file +**Result: Faster node recovery, reduced infrastructure load, improved cluster scalability with enterprise-grade +reliability** diff --git a/docs/ops_guide/repush.md b/docs/operations/data-management/repush.md similarity index 79% rename from docs/ops_guide/repush.md rename to docs/operations/data-management/repush.md index c4aae42d818..193a81b3826 100644 --- a/docs/ops_guide/repush.md +++ b/docs/operations/data-management/repush.md @@ -1,10 +1,3 @@ ---- -layout: default -title: Repush -parent: Operator Guides -permalink: /docs/ops_guide/repush ---- - # Repush In Venice, there are many things which happen under the hood as part of Full Pushes. Besides serving to refresh data, @@ -13,13 +6,15 @@ Full Pushes also serve to make certain configs take effect, and also more genera Many Venice users choose to periodically do Full Pushes for the sake of refreshing their data, in which case the auxiliary goals of Full Pushes end up being fulfilled organically. -In other cases, users do Full Pushes either infrequently, or not at all. In these cases, the operator may want to +In other cases, users do Full Pushes either infrequently, or not at all. In these cases, the operator may want to manually trigger a Repush, without involving the user. ## Use Cases + The various motivations for operator-driven Repushes are described in more details below. ### Reconfiguration + In Venice, some store configs can be changed live, while some others take effect when the next store-version is pushed. Configs which can be changed with a Repush (or a regular Full Push) include: @@ -32,64 +27,74 @@ Configs which can be changed with a Repush (or a regular Full Push) include: Having some configs be immutable within the scope of a store-version makes the development and maintenance of the system easier to reason about. In the past, the Venice team has leveraged Repushes to execute large scale migrations as new -modes were introduced (e.g., migrating from active-passive to active-active replication, or migrating from the -offline/bootstrap/online state model to the leader/follower state model). Repush enabled operators to roll out (and -occasionally roll back) these migrations in a way that was invisible to users of the platform. Although these migrations +modes were introduced (e.g., migrating from active-passive to active-active replication, or migrating from the +offline/bootstrap/online state model to the leader/follower state model). Repush enabled operators to roll out (and +occasionally roll back) these migrations in a way that was invisible to users of the platform. Although these migrations are behind us, it is possible that future migrations may also be designed to be executed this way. ### Sorting -When doing a Full Push (and also when Repushing), the data gets sorted inside the Map Reduce job, prior to pushing. The -Start of Push control message is annotated with the fact that the incoming data is sorted, which triggers an alternative + +When doing a Full Push (and also when Repushing), the data gets sorted inside the Map Reduce job, prior to pushing. The +Start of Push control message is annotated with the fact that the incoming data is sorted, which triggers an alternative code path within servers (and Da Vinci clients) in order to generate the storage files more efficiently. -When doing a Stream Reprocessing (SR), on the other hand, data does not get sorted. More generally, if the last Full -Push happened a while ago, and lots of nearline writes happened since then, there could be a large portion of the data +When doing a Stream Reprocessing (SR), on the other hand, data does not get sorted. More generally, if the last Full +Push happened a while ago, and lots of nearline writes happened since then, there could be a large portion of the data which is unsorted, and the operator may find that rebalance performance degrades as a result. -While it is not considered necessary to perform a Repush after every SR just for the sake of sorting, it can sometimes +While it is not considered necessary to perform a Repush after every SR just for the sake of sorting, it can sometimes be useful if the last SR or Full Push happened quite a while ago. ### Repair + In some cases where a datacenter (DC) suffered an outage, it is possible that pushes succeeded in some DCs and failed in -others. Alternatively, it's possible that correlated hardware failures resulted in some DC losing all replicas of some +others. Alternatively, it's possible that correlated hardware failures resulted in some DC losing all replicas of some partitions. In these cases, as long as one of the DCs is healthy, it can be used as the source of a repush to repair the data in other DCs. ### Time to Live (TTL) -It's usually required to evict stale entries from the store in order to achieve GDPR compliance or other business requirements. -The repush with TTL functionality will replay and scan through the entries in the current version, -and use store-level rewind time as TTL time to evict stale records based on the write timestamp. See [TTL](../user_guide/ttl) -for the comparison among various TTL options. + +It's usually required to evict stale entries from the store in order to achieve GDPR compliance or other business +requirements. The repush with TTL functionality will replay and scan through the entries in the current version, and use +store-level rewind time as TTL time to evict stale records based on the write timestamp. See [TTL](ttl.md) for the +comparison among various TTL options. ## Usage + The following Venice Push Job config is used to enable Repush functionality: + ``` source.kafka = true ``` ### Optional Configs + The following Venice Push Job configs provide more control over the Repush functionality: To specify which Kafka topic (i.e. which store-version) to use as the source of the Repush. If unspecified, it will default to the highest available store-version. + ``` kafka.input.topic = store-name_v7 ``` To specify which datacenter to pull from. If unspecified, it will default to one of the DCs which contains the highest available store-version. + ``` kafka.input.fabric = dc-name ``` To specify the address of the Kafka cluster to pull from. If unspecified, it will use the one returned by the controller of the datacenter chosen to pull from. + ``` kafka.input.broker.url = kafka-url:port ``` To specify the size of the offset range each mapper task will be responsible for. Each Kafka partition is fetched in parallel by multiple mappers, each taking care of distinct ranges. If unspecified, the default is 5M. + ``` kafka.input.max.records.per.mapper = 1000000 ``` @@ -97,17 +102,20 @@ kafka.input.max.records.per.mapper = 1000000 To specify whether to enable the Combiner of the Map Reduce job, in order to prune the shuffling traffic between mappers and reducers. This is an experimental functionality that has not been vetted for production usage yet. If unspecified, it defaults to false. + ``` kafka.input.combiner.enabled = true ``` To specify whether to enable TTL for the repush job. If unspecified, it defaults to false. + ``` repush.ttl.enable = true ``` -More details on how to configure TTL can be found in the [TTL](../user_guide/ttl) guide. + +More details on how to configure TTL can be found in the [TTL](ttl.md) guide. ## Future Work One potential enhancement would be to add support for changing the compression setting of a store, which currently -requires a Full Push, and which Repush does not support yet. \ No newline at end of file +requires a Full Push, and which Repush does not support yet. diff --git a/docs/ops_guide/system_stores.md b/docs/operations/data-management/system-stores.md similarity index 70% rename from docs/ops_guide/system_stores.md rename to docs/operations/data-management/system-stores.md index a87a9ed3395..73fe4918015 100644 --- a/docs/ops_guide/system_stores.md +++ b/docs/operations/data-management/system-stores.md @@ -1,14 +1,7 @@ ---- -layout: default -title: System Stores -parent: Operator Guides -permalink: /docs/ops_guide/system_stores ---- - # System Stores -Venice has several "system stores". These are used to store data and metadata needed by the system itself, in opposition -to "user stores" which store the data users are interested in. For the most part, the existence of these system stores +Venice has several "system stores". These are used to store data and metadata needed by the system itself, in opposition +to "user stores" which store the data users are interested in. For the most part, the existence of these system stores should not matter, but it can be useful for operators to know about them. This page explains the various types of system stores and their function. @@ -18,7 +11,7 @@ This page explains the various types of system stores and their function. Below are a few distinct types of system stores: | Type | Cardinality | Content | -|------------------------|----------------|---------------| +| ---------------------- | -------------- | ------------- | | Schema system store | 1 / deployment | Metadata only | | Global system store | 1 / deployment | Data | | Cluster system store | 1 / cluster | Data | @@ -26,13 +19,14 @@ Below are a few distinct types of system stores: ### Schema System Stores -These are stores created in just one of the clusters of a deployment. They contain no data at all (and therefore no +These are stores created in just one of the clusters of a deployment. They contain no data at all (and therefore no store-versions) and thus cannot be queried. They only serve to store schemas. These schemas are those which the system -needs for its own [internal protocols](https://venicedb.org/javadoc/com/linkedin/venice/serialization/avro/AvroProtocolDefinition.html). +needs for its own +[internal protocols](https://venicedb.org/javadoc/com/linkedin/venice/serialization/avro/AvroProtocolDefinition.html). -The schema system stores are used for the sake of protocol forward compatibility. When deploying Venice code, its jars -contain resource files for the current protocol versions as well as all previous protocol versions, but not future -protocol versions. Given that Venice is a distributed system, it is important to have the flexibility of deploying +The schema system stores are used for the sake of protocol forward compatibility. When deploying Venice code, its jars +contain resource files for the current protocol versions as well as all previous protocol versions, but not future +protocol versions. Given that Venice is a distributed system, it is important to have the flexibility of deploying different versions of the software across different nodes. When a node running a newer version of the software persists data written with a newer protocol version, and that data is then read by another node running an older version of the software which does not know about that newer protocol version, then it can query the schema of the appropriate system @@ -41,8 +35,8 @@ Schema Evolution). ### Global System Stores -These are stores created in just one of the clusters of a deployment, but in opposition to schema system stores, they -do contain data. Their global nature makes them convenient to store data about all clusters, but it also means that the +These are stores created in just one of the clusters of a deployment, but in opposition to schema system stores, they do +contain data. Their global nature makes them convenient to store data about all clusters, but it also means that the clusters are not isolated from one another as far as this kind of system store is concerned. Therefore, it is best for this type of system store to serve only in "best effort" purposes, so that if for any reason they become unavailable, critical operations can still continue. An example of this would be to store heuristics or other non-critical data. @@ -69,10 +63,10 @@ These are stores created once per user store, and which do contain data. These i Below is a list of actual system stores, along with their type and function (excluding schema stores): -| Name | Type | Function | -|----------------------------|-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Push Job Details Store | Global | Stores historical status and characteristics about all [Push Jobs](../user_guide/write_api/push_job.md) that ever ran. | -| Batch Job Heartbeat Store | Global | Stores heatbeats emitted from running push jobs, in order to detect if any push job terminated abruptly without cleaning itself up. | -| Participant Store | Cluster | Intended as a signaling mechanism for commands emitted by controllers, to be executed by servers, including kill job commands. | -| Da Vinci Push Status Store | Per-store | Stores per-instance consumption status of [Da Vinci Clients](../user_guide/read_api/da_vinci_client.md). Written to by DVC, and read by controllers, in order to determine whether push jobs have succeeded. | -| Meta Store | Per-store | Stores a copy of the store config (also found in Zookeeper). Written to by controllers, and read by DVC. Deprecated, will be removed eventually. | +| Name | Type | Function | +| -------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Push Job Details Store | Global | Stores historical status and characteristics about all [Push Jobs](../../user-guide/write-apis/batch-push.md) that ever ran. | +| Batch Job Heartbeat Store | Global | Stores heatbeats emitted from running push jobs, in order to detect if any push job terminated abruptly without cleaning itself up. | +| Participant Store | Cluster | Intended as a signaling mechanism for commands emitted by controllers, to be executed by servers, including kill job commands. | +| Da Vinci Push Status Store | Per-store | Stores per-instance consumption status of [Da Vinci Clients](../../user-guide/read-apis/da-vinci-client.md). Written to by DVC, and read by controllers, in order to determine whether push jobs have succeeded. | +| Meta Store | Per-store | Stores a copy of the store config (also found in Zookeeper). Written to by controllers, and read by DVC. Deprecated, will be removed eventually. | diff --git a/docs/user_guide/ttl.md b/docs/operations/data-management/ttl.md similarity index 76% rename from docs/user_guide/ttl.md rename to docs/operations/data-management/ttl.md index 1b5e94cbfc7..e4c36a284aa 100644 --- a/docs/user_guide/ttl.md +++ b/docs/operations/data-management/ttl.md @@ -1,32 +1,28 @@ ---- -layout: default -title: Time to Live (TTL) -parent: User Guides -permalink: /docs/user_guide/ttl ---- - # Time to Live (TTL) ## Use Cases + TTL can serve as a storage efficiency optimization for use cases where the data is transient in nature, and not useful beyond a certain age. TTL can also be useful in certain scenarios where the data should not be retained longer than prescribed for compliance -reasons. Note that for these scenarios, it is important to understand (as explained above) that the rewind time is the -minimum, not maximum, TTL. It is also advisable to schedule the periodic purging such that there is a margin of safety +reasons. Note that for these scenarios, it is important to understand (as explained above) that the rewind time is the +minimum, not maximum, TTL. It is also advisable to schedule the periodic purging such that there is a margin of safety in case of infra delays or failures. ## Usage + There are two ways to achieve TTL compliance, via [Repush with TTL](#repush-with-ttl) or via [empty push](#empty-push). The major differences between both approaches is the original data source and how real-time buffer is replayed. | | Source Data for TTL | Real-time buffer replay | -|-----------------|------------------------|-----------------------------------------------------------------------| +| --------------- | ---------------------- | --------------------------------------------------------------------- | | Empty push | None | Replay real-time buffer with `hybrid-rewind-seconds` config | | Repush with TTL | Existing version topic | Replay real-time buffer with `rewind.time.in.seconds.override` config | ## Repush with TTL -Repush with TTL runs as a VenicePushJob that reads the data in the existing version of the store, and writes it back as + +Repush with TTL runs as a VenicePushJob that reads the data in the existing version of the store, and writes it back as a new version. The TTL is enforced by the VenicePushJob, which only writes records that are younger than the specified TTL. Repush with TTL can also be configured and scheduled periodically to enforce TTL. @@ -34,25 +30,30 @@ The store-level rewind time (in hybrid store configs) is used as the default TTL specified by setting one of the following configs on the VenicePushJob: | Config | TTL behavior | -|------------------------------|----------------------------------------------------------------------| +| ---------------------------- | -------------------------------------------------------------------- | | `repush.ttl.seconds` | Records older than the specified value will expire | | `repush.ttl.start.timestamp` | Records that were written before the specified timestamp will expire | The `rewind.time.in.seconds.override` is a configurable value in push job (Default: 24 hours). This brings two major benefits: + 1. The repush job de-dupes writes to the same key, so that the servers need to ingest fewer events. 2. The repush job sorts the data, thus allowing servers to ingest in a more optimized way. ### How to enable + In order to enable repush with ttl, a VenicePushJob needs to be configured with the following configs: + ``` source.kafka = true repush.ttl.enable = true ``` -For more options related to repush, refer to the [Repush](../ops_guide/repush.md) guide. + +For more options related to repush, refer to the [Repush](repush.md) guide. ## Empty push + For Venice stores that receive nearline writes, it is possible to set up a periodic purge of records that are older than some threshold. @@ -61,15 +62,18 @@ store-version, after the push finishes, but before swapping the read traffic ove replay phase where the last N seconds of buffered real-time data is replayed. The replayed data gets overlaid on top of the data from the Full Push. -This can be leveraged in order to enforce TTL on the dataset. By scheduling periodic empty pushes and configuring how far back to replay the real-time data (N), one can control the -TTL parameters. The time to live is defined by N, which acts as a "minimum TTL", while the "maximum TTL" is N + delay -between each empty push. For example, if you schedule a daily empty push, and N = 6 days, then the oldest data in your -store will be at least 6 days old, and at most 7 days old. +This can be leveraged in order to enforce TTL on the dataset. By scheduling periodic empty pushes and configuring how +far back to replay the real-time data (N), one can control the TTL parameters. The time to live is defined by N, which +acts as a "minimum TTL", while the "maximum TTL" is N + delay between each empty push. For example, if you schedule a +daily empty push, and N = 6 days, then the oldest data in your store will be at least 6 days old, and at most 7 days +old. N can be set up by configuring the `hybrid-rewind-seconds` using venice-admin-tool. ### How to enable + At the moment, there are two ways to perform empty pushes: + 1. Via the `empty-push` command in the admin tool. 2. Via the Venice Push Job, executed from a Hadoop grid, but with an input directory that contains a file with no records in it. diff --git a/docs/operations/index.md b/docs/operations/index.md new file mode 100644 index 00000000000..aecf23f596c --- /dev/null +++ b/docs/operations/index.md @@ -0,0 +1,8 @@ +# Operations Guide + +Documentation for operating Venice. + +## Sections + +- [Data Management](data-management/repush.md) - Repush, TTL, and system stores +- [Advanced](advanced/p2p-bootstrapping.md) - P2P bootstrapping and data integrity diff --git a/docs/ops_guide/ops_guide.md b/docs/ops_guide/ops_guide.md deleted file mode 100644 index d7a62c3ea0e..00000000000 --- a/docs/ops_guide/ops_guide.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -layout: default -title: Operator Guides -has_children: true -permalink: /docs/ops_guide ---- -# Operator Guide - -This folder includes guides for Venice operators. Users should refer to Venice User Guide on how-to and the concepts. \ No newline at end of file diff --git a/docs/proposals/proposals.md b/docs/proposals/proposals.md deleted file mode 100644 index 102489fabdf..00000000000 --- a/docs/proposals/proposals.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -layout: default -title: Proposals -has_children: true -permalink: /docs/proposals ---- - -# Developer Guide - -This folder includes all the Venice Improvement Proposals. Read further details about the [VIP process](../dev_guide/how_to/design_doc.md). - -| VIP-# | Proposal | Status | -|---------------------|--------------------------------------------------------------------------------|------------------| -| [VIP-1](./vip-1.md) | Authentication Service API | Accepted | -| [VIP-2](./vip-2.md) | Removing Per Record Offset Metadata From Venice-Server Storage With Heartbeats | Under Discussion | -| [VIP-3](./vip-3.md) | Rust Server Read Path | Under Discussion | -| [VIP-4](./vip-4.md) | Store Lifecycle Hooks | Accepted | -| [VIP-5](./vip-5.md) | Facet Counting | Under Discussion | -| [VIP-6](./vip-6.md) | Venice on Kubernetes | Under Discussion | \ No newline at end of file diff --git a/docs/proposals/vip-1.md b/docs/proposals/vip-1.md deleted file mode 100644 index a3c42b0bedf..00000000000 --- a/docs/proposals/vip-1.md +++ /dev/null @@ -1,163 +0,0 @@ ---- -layout: default -title: VIP-1 Authentication Service API -parent: Proposals -permalink: /docs/proposals/vip-1 ---- - -# VIP-1: Authentication Service API - -* **Status**: _Accepted_ -* **Author(s)**: _Enrico Olivelli_ -* **Pull Request**: [PR 471](https://github.com/linkedin/venice/pull/471) -* **Release**: N/A - -## Introduction - -Currently, Venice doesn't provide a well-defined extensible way to authenticate clients, but it only supports -TLS based authentication, and it is hard coded. -This VIP proposes a new API to write plugins to authenticate clients. -The first follow-up work will be to implement a plugin to support JWT token based authentication. - -## Problem Statement - -Venice's services are based on HTTP/REST APIs, and we need a way to authenticate clients using standard -mechanisms like JWT tokens and OAuth2. -It means that all the services must perform authentication (and authorization) checks on each request. - -Therefore, we need a way to write plugins to perform authentication checks, this way it will be easy to -add more and more mechanisms in the future. - -Authentication mechanisms vary a lot from each other, and they are not only based on single steps; -we need to support at least only mechanisms that work well on HTTP and that can be performed in a single step, -like basic HTTP Authentication, that needs only to use an HTTP Header -or TLS based Authentication, that is based on the client certificate exchanged during the TLS handshake. - - -## Scope - -1. **What is in scope?** - - - Define an API to write plugins to perform authentication checks on Venice components (controller, server and router). - - The API must support single step authentication mechanisms (JWT, OAuth2, TLS client authentication). - - Support retrofitting the existing TLS mechanism as a plugin (without introducing the implementation). - - Ensure that the Authentication API is used by all the services and applied to every request. - - Ensure that the AuthorizerService is able to use the Authentication API to perform authorization checks. - -2. **What is out of scope?** - - - Remove legacy DynamicAccessController - - Implement a JWT plugin to support JWT token based authentication (this will be a follow-up work) - - Implement the TLS client certificate plugin - - Refactor the AuthorizerService APIs (even if some changes will be needed) - - Implement other authentication mechanisms (Kerberos, SAML, etc.) - - Modify the Admin Tools to support authentication (this will be a follow-up work) - - Deal with authentication against the PubSub broker (Kafka, Pulsar, etc.) - -## Project Justification - -This VIP is required in order to allow Venice user to use standard authentication mechanisms like JWT and OAuth2/OpenID Connect. - -For instance JWT, OAuth2 and OpenId connect are widely used by Apache Pulsar users, and introducing these features will -help the adoption of Venice in the Pulsar community. - -## Functional specification - -The core of the VIP is the AuthenticationService API, that is used by all the services to perform authentication checks. - -The AuthenticationService API is a Java interface that mandates the contract for the authentication plugins. - -The goal of the AuthenticationService is to map an HTTP request to a user identity, a **Principal**, that can be used by the AuthorizerService. - -As the AuthenticationService must work on all the VeniceComponents it won't have any hard dependency on the HTTP layer, -as in Venice we are using multiple technologies, depending on the Component. - -```java -public interface AuthenticationService extends Closeable { - - /** - * Maps an HTTP request to a Principal. - * Any unchecked exception thrown by this method will be logged and the request will be rejected. - * @param requestAccessor access the HTTP Request fields - * @return the Principal or null if the request is not authenticated - */ - default Principal getPrincipalFromHttpRequest(HttpRequestAccessor requestAccessor); - - /** - * Generic Wrapper over an HTTP Request. - */ - interface HttpRequestAccessor { - String getHeader(String headerName); - X509Certificate getCertificate(); - } - - /** - * Lifecycle method, called when the Venice component is initialized. - * @param veniceProperties the configuration of the component being initialized - * @throws Exception - */ - default void initialise(VeniceProperties veniceProperties) throws Exception; - - /** - * Lifecycle method, called when the Venice component is closed. - */ - default void close(); - -} -``` - -You configure the classname of the AuthenticationService with an entry `authentication.service.class`. -The AuthenticationService is initialised by passing the VeniceProperties read from the configuration file. - -## Proposed Design - -Most the work is about introducing the new API and refactoring the existing code to use it. -The legacy Authentication mechanism will initially be left untouched, but when you configure -the new AuthenticationService, the legacy mechanism will be disabled. - -AuthenticationService and AuthorizerService will kick-in in spite of the legacy DynamicAccessController. -And then the AuthenticationService the ACL checks will be performed by the AuthorizerService, -and not by the DynamicAccessController. - -In Venice clusters in which you configure AuthenticationService and AuthorizerService it is not expected -that all the current admin tools work, especially the ones that are based on the legacy DynamicAccessController (ACLs...). -It is out of the scope of this VIP to modify the admin tools. - -Multi-region support is not taken into account because we are only introducing a new API about Authentication, -it depends on every specific mechanism how to implement intra and inter region authentication. - -New Authentication mechanism may have an impact on performance depending on the technology user and the implementation. -It is out of the scope of this VIP to enter the details of the performance impact. -It is possible that in the future in order to support some authentication mechanisms we will need to introduce -an asynchronous API to perform authentication checks. - -## Development Milestones - -The implementation for this VIP introduces: -- the Java API -- the Controller implementation (loading the plugin and calling the API) -- the Router implementation (loading the plugin and calling the API) -- the Server implementation (loading the plugin and calling the API) -- some dummy plugins to test the API - -## Test Plan - -The implementation will be tested with unit tests and integration tests, main topics: -- AuthenticationService plugin lifecycle (boostrap, initialization, close) -- Verifying that the plugin is invoked by the Controller, Router and Server - -## References - -- [AuthenticationProvider API in Apache Pulsar](https://github.com/apache/pulsar/blob/master/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java) -- [Authentication Service Docs in Apache Pulsar](https://pulsar.apache.org/docs/3.0.x/security-authorization/) -- [OAuth2](https://oauth.net/2/) -- [JWT](https://jwt.io/introduction) - - - - - - - - - diff --git a/docs/proposals/vip-2.md b/docs/proposals/vip-2.md deleted file mode 100644 index c2c77d96b1b..00000000000 --- a/docs/proposals/vip-2.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -layout: default -title: VIP-2 Removing Per Record Offset Metadata From Venice-Server Storage With Heartbeats -parent: Proposals -permalink: /docs/proposals/vip-2 ---- - -# VIP-2: Removing Per Record Offset Metadata From Venice-Server Storage With Heartbeats - -* **Status**: _Under Discussion_ -* **Author(s)**: _Zac Policzer_ -* **Pull Request**: [PR 513](https://github.com/linkedin/venice/pull/513) -* **Release**: _N/A_ - -Not yet merged. Discussed in [PR 513](https://github.com/linkedin/venice/pull/513). \ No newline at end of file diff --git a/docs/proposals/vip-4.md b/docs/proposals/vip-4.md deleted file mode 100644 index 426f2467abe..00000000000 --- a/docs/proposals/vip-4.md +++ /dev/null @@ -1,278 +0,0 @@ ---- -layout: default -title: VIP-4 Store Lifecycle Hooks -parent: Proposals -permalink: /docs/proposals/vip-4 ---- - -# VIP-4: Store Lifecycle Hooks - -* **Status**: _Accepted_ -* **Author(s)**: _Felix GV_ -* **Pull Request**: [PR 881](https://github.com/linkedin/venice/pull/881) -* **Release**: _N/A_ - -## Introduction - -The [Venice Push Job](../user_guide/write_api/push_job.md) takes data from a grid and pushes it to a store in all -regions. This works fine in many cases, but there are some use cases where we would like to have greater control over -the steps of the process. This proposal is to add new configs and hooks which can be used both to monitor and control -the push job in a finer-grained manner than is possible today. In particular, this proposal focuses on the way that each -individual region is handled. - -Currently, the sequence of steps happening within a push job is as follows: - -![Push Job Steps](../assets/images/push_job_steps.drawio.svg) - -A few notes on the above diagram: - -* A new schema will be registered only if the data being pushed does not conform to any already known schema, and - the schema auto-registration config is enabled. If that config is disabled, then an unknown schema leads to job - failure. - -* The compression job is optional, and whether it runs or not depends on configurations. - -* The data push job writes to a store-version pub sub topic in one of the regions (typically the one which is local to - where the push job is running), but all regions start ingesting right away, as soon as the data push job begins - writing. For regions that are remote from the topic the data push job is writing to, the leader servers are performing - replication, while in the region which contains that topic, the replication is a no-op. - -* The `SERVING` step is also called "version swap". If there are Da Vinci Clients, they will ingest and swap on their - own, and the child controller of that region will wait until all DVC instances have swapped (started serving) before - enacting the region-level swap, after which the servers will also start serving the new store-version to clients which - perform remote queries. That is why the "DVC Read Traffic SERVING" step is a dependency for the "Server Read Traffic - SERVING" step. - -## Problem Statement - -Pushing to all regions in parallel makes the push faster, but it also means that if the push causes an issue, the impact -is going to be global (affecting all regions). It would be desirable to have certain checks and balances that reduce the -blast radius in cases where the content of the push causes issues. Examples of such issues include: - -* Data quality issues: - * Incomplete data (due to issues in data generation logic, or in upstream datasets). - * Change in semantics (e.g. some embeddings trained by a new ML model / weights / params / etc. are incompatible with - that used at inference-time). -* Schema-related issues: - * Some optional fields which used to always be populated get deleted (or get populated with null) and the reading app - fails due to lack of null checking. This kind of app-side bug can happen even though a schema evolution is fully - compatible. -* Infra issues: - * Larger payloads take more resources, resulting in lack of capacity and thus latency degradation. - * Certain types of yet-unknown infra bugs are somehow triggered by a data push. - -Tighter control of how the new version of the dataset is deployed to each region could allow us to catch issues while -only one region is affected, and abort deploying to other regions. See Scope, below, for specific examples of flow -control strategies. - -In addition, we would like to make it easier to integrate the push job into proprietary monitoring systems such that -each region getting data deployed to it results in events getting emitted or other observability actions. - -## Scope - -This proposal is about full push jobs. Incremental pushes and nearline writes are out of scope. At the time of -submitting this proposal, it is undetermined whether this work will apply to stream reprocessing jobs. In terms of -priority, we care mostly about supporting full pushes from offline grids, and it may be fine to leave stream -reprocessing out of scope, although depending on the design details we choose, we may be able to support stream -reprocessing "for free" as well (i.e. if the hooks are executed in the controller). Incremental Push is out of scope of -this proposal. - -The goal is for lifecycle hooks to achieve the following use cases: - -* Orchestrate how data is served in each region, including: - * Ensuring a minimum delay (e.g. 1 hour) between each region beginning to serve the new store-version. - * Delaying the swapping of a new store-version to be within some time of day (e.g. during "business hours"). - * Performing custom health checks on the client applications to ensure that their key metrics are still healthy within - a region where a new store-version was swapped, before proceeding to more regions. Based on the outcome of this - check: - * Having the ability to abort the swapping of a store-version to further regions. - * Having the ability to rollback to the previous store-version in regions that already swapped. -* Trigger informational events in proprietary monitoring systems after important lifecycle milestones are completed. - -The above use cases all are operator-centric, and so (at least for now) there is no concern of making it very ergonomic -for Venice users to register new hooks or evolve old hooks dynamically. The general expectation is that there would be -a small number of hooks maintained by the Venice operators and that it's ok for hooks to be bundled and upgraded -alongside the Venice components. In cases where Venice users need to customize hook behaviors, that could be achieved -via store-level configs passed into hooks, and there is no need to provide the flexibility of letting users register -whole new hook implementations. - -## Project Justification - -The cost of having global impact in the case of issues mentioned above is too high, and we would like to provide -first-class options to reduce the blast radius. Building this within Venice itself will make it easier to automate these -methodologies, thus reducing toil for users. - -## Functional specification - -The proposed API for this functionality is described in code here: - -[//]: # (Got to remove the /venice prefix before pushing to main on linkedin/venice, otherwise links will be broken...) - -* [StoreLifecycleHooks](/venice/javadoc/com/linkedin/venice/hooks/StoreLifecycleHooks.html), which is the main part of this - proposal. -* [StoreLifecycleEventOutcome](/venice/javadoc/com/linkedin/venice/hooks/StoreLifecycleEventOutcome.html), which is the signal - returned by some hooks to indicate that a given step should proceed or abort. -* [StoreVersionLifecycleEventOutcome](/venice/javadoc/com/linkedin/venice/hooks/StoreVersionLifecycleEventOutcome.html), which - is the signal returned by some other hooks which need more fine-grained control over the workflow. In addition to - proceeding and aborting, this also provides the option to wait, which tells the hooks framework to try invoking the - hook again later, and rollback, which tells the framework to rollback to the previous store-version in all regions. -* [JobStatusQueryResponse](/venice/javadoc/com/linkedin/venice/controllerapi/JobStatusQueryResponse.html), which is the payload - returned by the `/job` controller endpoint, is extended to include status update timestamps. This will be populated by - the child controller to indicate the time when its own individual status last changed, and the parent controller will - aggregate these into a map keyed by region. All hooks which return the `StoreVersionLifecycleEventOutcome` will have - access to this payload in their input, so that they can make decisions based on the status of each region. The time - when the status was last updated for a given region is useful in order to achieve the use case of a hook which injects - a delay between each region swap. The code change to support these extra timestamps is included in this VIP, to - demonstrate feasibility and present the proposed algorithm (see `OfflinePushStatus::getStatusUpdateTimestamp`). - -## Proposed Design - -The main design consideration is where to execute the hooks. At a high level, there are three options: - -1. Within the push job. -2. Within the parent controller (chosen option). -3. Within the child controllers. - -It is also possible to consider invoking some hooks in one of these location while other hooks would be executed -elsewhere. The table below summarizes the feasibility and tradeoffs for each of the proposed hooks: - -| Hook function name | Actions | CC | PC | VPJ | -|---------------------------------------------:|:----------:|:---:|:---:|:---:| -| `validateHookConfig` | ➡️ ☠️ | ❌ | ✅ | ❌ | -| `preStartOfPushJob` | ➡️ ☠️ | ❌ | ✅ | ✅ | -| `postStartOfPushJob` | None | ❌ | ✅ | ✅ | -| `preSchemaRegistration` | ➡️ ☠️ | ❌ | ✅ | 1️⃣ | -| `postSchemaRegistration` | None | ✅ | ✅ | 1️⃣ | -| `preStoreVersionCreation` | ➡️ ☠️ ✋ ↩️ | ✅ | ✅ | ✅ | -| `postStoreVersionCreation` | None | ✅ | ✅ | ✅ | -| `preStartOfStoreVersionIngestionForDaVinci` | ➡️ ☠️ ✋ ↩️ | 2️⃣ | 2️⃣ | 2️⃣ | -| `postStartOfStoreVersionIngestionForDaVinci` | ➡️ ☠️ ✋ ↩️ | 2️⃣ | 2️⃣ | 2️⃣ | -| `postStoreVersionLeaderReplication` | None | ✅ | ❌ | ❌ | -| `preStoreVersionSwap` | ➡️ ☠️ ✋ ↩️ | ✅ | ✅ | ✅ | -| `postStoreVersionSwap` | ➡️ ☠️ ✋ ↩️ | ✅ | ✅ | ✅ | -| `preEndOfPushJob` | ➡️ ☠️ ✋ ↩️ | ✅ | ✅ | ✅ | -| `postEndOfPushJob` | None | ✅ | ✅ | ✅ | - -**Legend:** - -* The Actions column represent which control mechanism is available to each hook: - * ➡️ Proceed: Move forward with this step. - * ☠️ Abort: Cancel this step (and as a consequence, short-circuit any future step that would come after this one). - * ✋ Wait: Let the hooks framework re-run this step later (i.e. 1 minute later, by default). - * ↩️ Rollback: Let the store go back to the store-version it had prior to beginning the push job (in all regions). - -* The last three columns are the feasibility of implementing this hook in a given component: - * **CC** Child Controller. - * **PC** Parent Controller. - * **VPJ** Venice Push Job. - * ✅ It is feasible to implement this hook within this component (without unreasonable complexity). - * ❌ It is NOT feasible to implement this hook within this component (without unreasonable complexity). - * 1️⃣ Schema registration hooks in push jobs could only be invoked in cases where auto-registration is enabled and - the new schema originates from the push job itself, whereas schema registrations which are performed directly on the - controller could not trigger the hook. - * 2️⃣ The hook for the start of ingestion for Da Vinci Clients is tricky for a few reasons. The start of ingestion is - controlled by updating the Meta Store, which is a non-replicated system store updated by child controllers, so those - must be involved (either by running the hook there in the first place, or by having some mechanism that enables the - parent or VPJ to inform the child controllers of when the system store is eligible for getting updated, such as by - adding a new field to the `AddVersion` admin channel command). However, see Rollback Support below for why running - hooks in the child controller may be insufficient. - -### Rollback Support - -In order to support the ability for the hooks which return the `StoreVersionLifecycleEventOutcome` to rollback, the most -natural way to achieve this is likely to involve the parent controller, either by having those hooks run there in the -first place, or by having a propagation mechanism to it: - -* If the hooks are executed in the child controller, the propagation mechanism might be to extend the job status check - which the parent controller does periodically in order for the child to inform the parent of the need to rollback, or - else build a new mechanism for the child to directly interact with the parent (or even with other child controllers - directly...). - -* If the hooks are executed in VPJ, then it would need to interact with the parent via the controller client to trigger - the rollback. - -### Configs - -There needs to be new configs: - -* `venice.store.lifecycle.hooks`: Comma-separated list of FQCN of the hook implementations to load. - -* `venice.store.lifecycle.hooks.threads`: Number of threads used by the hooks framework to execute all hooks. - -* `venice.store.lifecycle.hooks.timeout.seconds`: Max duration allowed for a hook before the hooks framework interrupts - it. - -* `venice.store.lifecycle.hooks.wait.interval.seconds`: The time to wait before re-invoking a hook which returned `WAIT` - (default: 60 seconds). - -* `venice.store.lifecycle.hooks.configs.`: Any number of configs to be passed (after clipping everything - before the `` part) into the hooks constructor. - -In addition, the store config will get a new `Map` of store-level config overrides. Those configs are -"[stringly-typed](https://wiki.c2.com/?StringlyTyped)", rather than strongly-typed, since we are not aware of the -configs needed by each hook at compile-time, and we therefore cannot shape the definition of the store config schema -accordingly. This issue is mitigated via the `validateHookConfig`, which can be used to prevent invalid configs from -entering the system. - -### Metrics - -There needs to be new metrics to monitor hook health. Each new metric will be per function and per registered hooks -class (i.e. 13 functions per class if we implement all of them, or less if we cut the scope). For each class/function -hook, there will be: - -* the occurrence rate of: - * hook invocations - * each hook return signal (for the functions that return something other than `void`) - * failures (exceptions) - * timeouts -* the time spend waiting in queue before being executed (which will be useful to determine if the thread pool count is - under-provisioned) - -N.B.: Implementing metrics from within VPJ is a bit more complicated, whereas controllers already have the ability to -emit metrics. - -### Design Recommendation - -Running hooks in the parent controller is probably most straightforward. The only issue is the inability to support the -`postStoreVersionLeaderReplication` hook, but that one is not critical and could be left out of scope. - -Concretely, choosing the parent controller path would work like this: - -1. The parent controller would invoke the hooks for `preStartOfStoreVersionIngestionForDaVinci` and for - `preStoreVersionCreation` for each of the regions. If all hooks respond with `PROCEED` then it's essentially a normal - push, otherwise it would configure the `AddVersion` command sent to the admin channel with the appropriate inclusions - in `targetedRegions` (depending on the result of `preStoreVersionCreation`) and in a new field for controlling the - DVC ingestion (depending on the result of `preStartOfStoreVersionIngestionForDaVinci`) which the child controller - would honor by holding off on writing to the Meta Store. - -2. The parent controller would create the new store-version with `versionSwapDeferred = true`. - -3. When the parent controller sees that a region has completed ingestion, it would invoke the `preStoreVersionSwap` - hook for that region, and if the hook responds with `PROCEED`, then it would send an admin command targeted for that - region to swap the current version. - -4. If at any stage the hooks respond with `ROLLBACK` then the parent controller would send more admin channel commands - to do the swap in the reverse direction. - -**This recommendation has been accepted, after design reviews, hence we will implement hooks in the parent controller.** - -## Development Milestones - -At a high-level: - -1. Evolve the admin channel protocol. -2. Implement the child controller changes for dealing with the admin channel changes. -3. Implement the parent controller changes to support the hooks framework and the orchestration described above. - -More fine-grained plan TBD after finalizing the design. - -## Test Plan - -The first hooks will be built such that they are no-op by default, and require a hook config to enable them. That hook -config will be left disabled in the global config, and will be tested at small scale via the store-level overrides. - -After store-level testing and stabilization is satisfactory, we will begin enabling them globally. - -## References - -N/A. \ No newline at end of file diff --git a/docs/quickstart/quickstart.md b/docs/quickstart/quickstart.md deleted file mode 100644 index 4e4006f8681..00000000000 --- a/docs/quickstart/quickstart.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -layout: default -title: Quickstart -has_children: true -permalink: /docs/quickstart ---- -# Quickstart - -This folder includes Venice quickstart guides. \ No newline at end of file diff --git a/docs/quickstart/quickstart_data_integrity_validation.md b/docs/quickstart/quickstart_data_integrity_validation.md deleted file mode 100644 index 1d76f821040..00000000000 --- a/docs/quickstart/quickstart_data_integrity_validation.md +++ /dev/null @@ -1,128 +0,0 @@ ---- -layout: default -title: Venice Data Integrity Validation -parent: Quickstart -permalink: /docs/quickstart/quickstart_data_integrity_validation ---- - - -# Venice Data Integrity Validation - -### What is Data Integrity Validation (DIV)? -- Validate and detect errors in transmitted (Kafka) data. -- The sender (VeniceWriter) calculates and includes the DIV metadata. -- The receiver recalculates and verifies the DIV metadata on arrival. - -### What does DIV metadata look like? -- Sender transmits data in segments. -- Sender decides when to close the current segment and start a new one. -- A segment contains multiple Kafka records - * Start of Segment (SOS) - * End of Segment (EOS) - * Control or data messages between -- GUID - * One unique ID per-sender (VeniceWriter). - * GUID changes if VeniceWriter or the server restarts. -- Sequence number - * Within a segment, the sequence number starts at 0. - -### Data integrity issues in Venice: -- A segment starts from a non-zero sequence number (UNREGISTERED_PRODUCER). -- A gap exists between or within segments (MISSING). -- Data within a segment is corrupted (CORRUPT). -- Producers have produced duplicate messages, which is expected due to retries (DUPLICATE). - -### Different behaviors before and after End Of Push (EOP) -- End Of Push (EOP) is a control message in the Kafka topic sent once per partition at bulk load end, after all data producers come online. -- When pushing a new store version, if DIV detects fatal data validation issues (including UNREGISTERED_PRODUCER, MISSING, and CORRUPT) while consuming batch push data before EOP, the ingestion task errors and aborts. -- If issues appear after receiving EOP, the ingestion task continues, but DIV logs a warning about data integrity issues. - -### Data structures -- Sender - VeniceWriter/Segment -- Data - ProducerRecord/KafkaMessageEnvelope/ProducerMetadata -- Receiver - ConsumptionTask/StoreIngestionTask - -### Persistence -- DIV needs to be checkpointed for several reasons: - * DIV state would be lost on server restarts leading to potential data integrity issues undetected (inaccuracy) - * Otherwise, without checkpointing, DIV needs to rebuild its state from the beginning of the topic on every restart (inefficiency) - * In case of a crash or restart, we need to know where to resume validation from (last checkpoint). - -### Why Two Separate DIV validators? -- Two state pipeline -- Stage 1: Consumer Thread - * Kafka → Consumer Thread → Validation → Queue → ... - * Reads messages from Kafka topics - * Performs validation using Consumer DIV (RT or remote VT) - * Queues messages to StoreBufferService - -- Stage 2: Drainer Thread - * ... → Queue → Drainer Thread → Validation → Storage Engine - * Dequeues messages from StoreBufferService - * Performs validation using drainer DIV (all topics) - * Persists data to storage engine - * Checkpoints offsets to disk. - -- Consumer is always ahead of drainer - * The consumer thread reads from Kafka faster than the drainer can persist to disk. There's buffering in between. - * Kafka producer buffers (for leaders) - * StoreBufferService queue etc. - * Time T1: Consumer validates message at offset 1000 → Consumer DIV state = offset 1000 - * Time T2: Consumer validates message at offset 2000 → Consumer DIV state = offset 2000 - * Time T3: Drainer persists message at offset 1000 → Drainer DIV state = offset 1000 - * If we only had one consumer DIV, we couldn't checkpoint the correct state to disk. - -- DIV State Must Match Persisted Data - * The DIV checkpoint saved to disk must exactly match what's actually persisted in the storage engine. Since the drainer is responsible for persistence, only the Drainer DIV state can be safely checkpointed. - -- Leader Double Validation - * Consumer DIV (in consumer thread): Validates RT messages before producing to VT - * Drainer DIV (in drainer thread): Validates same messages again before persisting (unnecessary) - -### DIV in State Transitions (per-partition) -- **OFFLINE -> STANDBY:** DIV state restoration. - The DIV state is restored from the persisted OffsetRecord (drainerDiv). In STANDBY mode, the DIV continues to validate incoming messages against the restored state and keeps it updated. -- **STANDBY -> OFFLINE:** The DIV state is cleared immediately without being persisted to disk. - Any DIV state accumulated since the last checkpoint is lost The system relies on periodic checkpoints during consumption, not on-demand checkpoints during state transitions. This design choice prioritizes fast un-subscription over preserving the absolute latest DIV state. -- **STANDBY -> LEADER:** Wipes all DIV state for the partition in the consumer DIV. - Copies the producer states from the drainer's DIV validator to the consumer DIV. -- **LEADER -> STANDBY:** The drainer DIV maintains producer states that have been validated and persisted. - If this replica becomes leader again, it will clone the drainer DIV state. - -### Kafka Log Compaction impacts DIV in Venice -- When Kafka log compaction runs, it: - * Deletes duplicate records (identified by the key) and keeps only the latest record. - * Creates **gaps** in sequence numbers that DIV would normally flag as data loss. - * Venice uses a **log compaction delay threshold** to distinguish between: - * **Real data loss** (missing messages within the compaction window) - * **Expected gaps** (missing messages beyond the compaction window) - * Example: - * T0: Producer sends messages with seq# 1, 2, 3, 4, 5 - * T1: Kafka stores all messages - * T2: min.compaction.lag.ms = 24 hours passes - * T3: Kafka compaction runs, deletes messages 2, 3 (duplicate keys) - * T4: Consumer reads: seq# 1, 4, 5 (gap detected!) - * DIV Check: - * Last message timestamp: T0 - * Current time: T4 (> 24 hours later) - * elapsedTime >= min.compaction.lag.ms? YES - * Result: TOLERATE the gap (expected compaction) - * T4: Consumer reads: seq# 1, 4, 5 (gap detected!) - * elapsedTime < min.compaction.lag.ms? YES - * Result: THROW EXCEPTION (data loss!) - -### DIV cleanup mechanism -- Prevents DIV from accumulating unbounded producer state by automatically expiring and removing old producer tracking information. -- Without cleanup, this state grows indefinitely - * Large memory footprint - State accumulates in heap - * Checkpoint overhead - All producer states are persisted to disk on every offset sync -- Solution: Time-Based Expiration - * **MaxAge** allows Venice to automatically expire old producer state that's no longer relevant. - * Only applied to drainer DIV (not consumer DIV, which is transient) - * For hybrid stores, maxAge is >= rewind time. - * Can be disabled by setting to -1. -- How it works: - * When checkpointing offsets, DIV clears expired state. - * When loading state from disk (e.g., after restart), maxAge is also applied. This prevents loading stale state after a restart. - * Why MaxAge ≥ Rewind Time for Hybrid Stores? - * Support RT replay, otherwise it doesn't recognize producer’s div state in RT. \ No newline at end of file diff --git a/docs/user_guide/learn_more.md b/docs/resources/learn-more.md similarity index 76% rename from docs/user_guide/learn_more.md rename to docs/resources/learn-more.md index 64df4e0baf0..72cf824ad19 100644 --- a/docs/user_guide/learn_more.md +++ b/docs/resources/learn-more.md @@ -1,43 +1,48 @@ ---- -layout: default -title: Learn More -parent: User Guides -permalink: /docs/user_guide/learn_more ---- - # Learn More -This page lists the Venice content published online. Please keep in mind that older content reflects an earlier phase of + +This page lists the Venice content published online. Please keep in mind that older content reflects an earlier phase of the project and may not be entirely correct anymore. ## Posts + The following blog posts have previously been published about Venice: -- 2015: [Prototyping Venice: Derived Data Platform](https://engineering.linkedin.com/distributed-systems/prototyping-venice-derived-data-platform) -- 2017: [Building Venice with Apache Helix](https://engineering.linkedin.com/blog/2017/02/building-venice-with-apache-helix) -- 2017: [Building Venice: A Production Software Case Study](https://engineering.linkedin.com/blog/2017/04/building-venice--a-production-software-case-study) -- 2017: [Venice Hybrid: Doing Lambda Better](https://engineering.linkedin.com/blog/2017/12/venice-hybrid--doing-lambda-better) +- 2015: + [Prototyping Venice: Derived Data Platform](https://engineering.linkedin.com/distributed-systems/prototyping-venice-derived-data-platform) +- 2017: + [Building Venice with Apache Helix](https://engineering.linkedin.com/blog/2017/02/building-venice-with-apache-helix) +- 2017: + [Building Venice: A Production Software Case Study](https://engineering.linkedin.com/blog/2017/04/building-venice--a-production-software-case-study) +- 2017: + [Venice Hybrid: Doing Lambda Better](https://engineering.linkedin.com/blog/2017/12/venice-hybrid--doing-lambda-better) - 2018: [Venice Performance Optimization](https://engineering.linkedin.com/blog/2018/04/venice-performance-optimization) -- 2021: [Taming memory fragmentation in Venice with Jemalloc](https://engineering.linkedin.com/blog/2021/taming-memory-fragmentation-in-venice-with-jemalloc) -- 2022: [Supporting large fanout use cases at scale in Venice](https://engineering.linkedin.com/blog/2022/supporting-large-fanout-use-cases-at-scale-in-venice) -- 2022: [Open Sourcing Venice – LinkedIn’s Derived Data Platform](https://engineering.linkedin.com/blog/2022/open-sourcing-venice--linkedin-s-derived-data-platform) +- 2021: + [Taming memory fragmentation in Venice with Jemalloc](https://engineering.linkedin.com/blog/2021/taming-memory-fragmentation-in-venice-with-jemalloc) +- 2022: + [Supporting large fanout use cases at scale in Venice](https://engineering.linkedin.com/blog/2022/supporting-large-fanout-use-cases-at-scale-in-venice) +- 2022: + [Open Sourcing Venice – LinkedIn’s Derived Data Platform](https://engineering.linkedin.com/blog/2022/open-sourcing-venice--linkedin-s-derived-data-platform) - 2023: [Public CI](https://blog.venicedb.org/public-ci) - 2023: [Stable Releases](https://blog.venicedb.org/stable-releases) - 2024: [VeniceCon 2024](https://blog.venicedb.org/venicecon-2024) - 2024: [Stable Release 0.4.263](https://blog.venicedb.org/stable-release-0-4-263) -Follow us on [HashNode](http://blog.venicedb.org) or [subscribe to the newsletter](https://blog.venicedb.org/newsletter) +Follow us on [HashNode](http://blog.venicedb.org) or [subscribe to the newsletter](https://blog.venicedb.org/newsletter) to hear about it when new posts are published. ## Talks + The following talks have been given about Venice: -[//]: # (- 2018: [Venice with Apache Kafka & Samza](https://www.youtube.com/watch?v=Usz8E4S-hZE)) -- 2019: [People You May Know: Fast Recommendations over Massive Data](https://www.infoq.com/presentations/recommendation-massive-data/) +[//]: # "- 2018: [Venice with Apache Kafka & Samza](https://www.youtube.com/watch?v=Usz8E4S-hZE)" + +- 2019: + [People You May Know: Fast Recommendations over Massive Data](https://www.infoq.com/presentations/recommendation-massive-data/) - 2019: [Enabling next generation models for PYMK Scale](https://www.youtube.com/watch?v=znd-Q6IvCqY) - 2022: Open Sourcing Venice - [Strange Loop 2022](https://www.youtube.com/watch?v=pJeg4V3JgYo) (original) @@ -46,11 +51,13 @@ The following talks have been given about Venice: - [QCon London 2023](https://www.infoq.com/presentations/derived-data/) (original) - [University lecture at ETS](https://www.youtube.com/watch?v=p1LtVo_1Q7A) (in French, includes Q&A) - 2023: [Partial Updates in Venice](https://www.youtube.com/watch?v=WlfvpZuIa6Q&t=3880s) -- 2023: [When Only the Last Writer Wins We All Lose: Active-Active Geo-Replication in Venice](https://www.youtube.com/watch?v=jfbg6IUgVlI) +- 2023: + [When Only the Last Writer Wins We All Lose: Active-Active Geo-Replication in Venice](https://www.youtube.com/watch?v=jfbg6IUgVlI) - 2023: [The Journey to a Million Ops / Sec / Node in Venice](https://www.infoq.com/presentations/scalable-low-latency/) - 2024: Lessons Learned from Building LinkedIn’s AI Data Platform - [QCon London 2024](https://www.infoq.com/presentations/ai-venice/) (original) - - [VeniceCon 2024: part 1](https://www.youtube.com/watch?v=PmjGEXek--s), [part 2](https://www.youtube.com/watch?v=PlCvF9C2RAU) + - [VeniceCon 2024: part 1](https://www.youtube.com/watch?v=PmjGEXek--s), + [part 2](https://www.youtube.com/watch?v=PlCvF9C2RAU) - [University lecture at ETS](https://www.youtube.com/watch?v=Vkazja71BkA) (in French, includes Q&A) - 2024: [TLA+ @ LinkedIn: Ambry and Venice](https://www.youtube.com/watch?v=Jz0J5N77QKk) - 2024: [What Venice Shipped in the Last Year](https://www.youtube.com/watch?v=5pVUKUvcXyg) @@ -58,18 +65,18 @@ The following talks have been given about Venice: - 2024: [Fast Client: Venice Next-Gen Read Path](https://www.youtube.com/watch?v=g2FxkEQU4P8) - 2025: [Da Vinci: Pumping Streams into RocksDB and DuckDB](https://www.youtube.com/watch?v=hc0pgvnr3fQ) -Follow us on [YouTube](https://www.youtube.com/@venicedb) to hear about it when new content is published on that +Follow us on [YouTube](https://www.youtube.com/@venicedb) to hear about it when new content is published on that platform. ## Podcasts + The following interviews have been given about Venice: | Year | Channel | Youtube | Apple | Spotify | -|------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------| +| ---- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------- | | 2022 | JUXT Cast | [![](../assets/icons/youtube-icon.svg)](https://www.youtube.com/watch?v=4QBW1Pa_oIk&list=PLfBAF5hZLVGkpALZDItoMbMrp3u3TWwQc&index=1) | [![](../assets/icons/apple-podcasts-icon.svg)](https://podcasts.apple.com/us/podcast/strange-loop-edition-a-chat-with-felix-gv/id1471141263?i=1000583323923) | [![](../assets/icons/spotify-icon.svg)](https://open.spotify.com/episode/2A7irnn0KTefvOMaJ4ks96?si=zS2JMR14SXqC-yjJEq9W0w) | | 2022 | SaaS Developer Community | [![](../assets/icons/youtube-icon.svg)](https://www.youtube.com/watch?v=7KVw13ia6Rs&list=PLfBAF5hZLVGkpALZDItoMbMrp3u3TWwQc&index=2) | | | | 2023 | Monday Morning Data Chat | [![](../assets/icons/youtube-icon.svg)](https://www.youtube.com/watch?v=wxo6irP8QYU&list=PLfBAF5hZLVGkpALZDItoMbMrp3u3TWwQc&index=3) | [![](../assets/icons/apple-podcasts-icon.svg)](https://podcasts.apple.com/us/podcast/monday-morning-data-chat/id1565154727?i=1000604537042) | [![](../assets/icons/spotify-icon.svg)](https://open.spotify.com/episode/1V102KRPO2D3HoS7xNQLJ6?si=fDunwgUnQhev_OZJA2eiMw) | | 2023 | Software Misadventures | [![](../assets/icons/youtube-icon.svg)](https://www.youtube.com/watch?v=aqsacLCQxaY&list=PLfBAF5hZLVGkpALZDItoMbMrp3u3TWwQc&index=4) | [![](../assets/icons/apple-podcasts-icon.svg)](https://podcasts.apple.com/us/podcast/software-misadventures/id1542480882?i=1000636756076) | [![](../assets/icons/spotify-icon.svg)](https://open.spotify.com/episode/3sbC4lVKBt8ZvM2Lt0DedK?si=KdlmlL7WQFqJNC2RIc1e_g) | | 2024 | The Geek Narrator | [![](../assets/icons/youtube-icon.svg)](https://www.youtube.com/watch?v=D6gZKM4Jnk4&list=PLfBAF5hZLVGkpALZDItoMbMrp3u3TWwQc&index=5) | [![](../assets/icons/apple-podcasts-icon.svg)](https://podcasts.apple.com/us/podcast/the-geeknarrator/id1619407689?i=1000657888486) | [![](../assets/icons/spotify-icon.svg)](https://open.spotify.com/episode/5P68vP7pBF8Q8ka0jni9S0?si=ZX_h29l4S7W-EkvfhMHR-Q) | | 2025 | InfoQ | [![](../assets/icons/youtube-icon.svg)](https://www.youtube.com/watch?v=Ey4kvQzkNsM&list=PLfBAF5hZLVGkpALZDItoMbMrp3u3TWwQc&index=6) | [![](../assets/icons/apple-podcasts-icon.svg)](https://podcasts.apple.com/us/podcast/the-infoq-podcast/id1106971805?i=1000699480482) | [![](../assets/icons/spotify-icon.svg)](https://open.spotify.com/episode/7z2JYjksn32zfgHFGuP5R8) | - diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md new file mode 100644 index 00000000000..9dbf3fe2d81 --- /dev/null +++ b/docs/user-guide/index.md @@ -0,0 +1,14 @@ +# User Guide + +Documentation for Venice users. + +![API Overview](../assets/images/api_overview.drawio.svg) + +## Sections + +- [Write APIs](write-apis/index.md) - Batch, streaming, and online writes +- [Read APIs](read-apis/index.md) - Thin, Fast, and Da Vinci clients + +## Additional Resources + +For foundational concepts and common patterns, see the [Getting Started](../getting-started/index.md) guide. diff --git a/docs/user_guide/read_api/da_vinci_client.md b/docs/user-guide/read-apis/da-vinci-client.md similarity index 68% rename from docs/user_guide/read_api/da_vinci_client.md rename to docs/user-guide/read-apis/da-vinci-client.md index 9bf8710d734..a2afc8e1850 100644 --- a/docs/user_guide/read_api/da_vinci_client.md +++ b/docs/user-guide/read-apis/da-vinci-client.md @@ -1,18 +1,12 @@ ---- -layout: default -title: Da Vinci Client -parent: Read APIs -grand_parent: User Guides -permalink: /docs/user_guide/read_api/da_vinci_client ---- - # Da Vinci Client -This allows you to eagerly load some or all partitions of the dataset and perform queries against the resulting local + +This allows you to eagerly load some or all partitions of the dataset and perform queries against the resulting local cache. Future updates to the data continue to be streamed in and applied to the local cache. ## Record Transformer (DVRT) -The Record Transformer lets you hook into Da Vinci's data ingestion process to react to every record change in real-time. +The Record Transformer lets you hook into Da Vinci's data ingestion process to react to every record change in +real-time. ### What Does It Do? @@ -23,17 +17,24 @@ The Record Transformer lets you hook into Da Vinci's data ingestion process to r ### Quick Start Guide #### Step 1: Implement the Interface -Extend [DaVinciRecordTransformer](https://github.com/linkedin/venice/blob/main/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformer.java) and implement: -- `transform` - Transform data before local persistence, returns [DaVinciRecordTransformerResult](https://github.com/linkedin/venice/blob/main/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerResult.java): +Extend +[DaVinciRecordTransformer](https://github.com/linkedin/venice/blob/main/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformer.java) +and implement: + +- `transform` - Transform data before local persistence, returns + [DaVinciRecordTransformerResult](https://github.com/linkedin/venice/blob/main/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerResult.java): - `UNCHANGED` - Keep original value - - `TRANSFORMED` - Use new transformed value + - `TRANSFORMED` - Use new transformed value - `SKIP` - Drop this record entirely - `processPut` - Handle record updates - `processDelete` - Handle deletions (optional) #### Step 2: Configure and Register -Build a [DaVinciRecordTransformerConfig](https://github.com/linkedin/venice/blob/main/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerConfig.java) and register it: + +Build a +[DaVinciRecordTransformerConfig](https://github.com/linkedin/venice/blob/main/clients/da-vinci-client/src/main/java/com/linkedin/davinci/client/DaVinciRecordTransformerConfig.java) +and register it: ```java DaVinciRecordTransformerConfig config = new DaVinciRecordTransformerConfig.Builder() @@ -44,8 +45,8 @@ DaVinciConfig daVinciConfig = new DaVinciConfig(); daVinciConfig.setRecordTransformerConfig(config); ``` -**For Custom Constructor Parameters:** -If you need to pass additional parameters to your transformer's constructor beyond the default ones provided: +**For Custom Constructor Parameters:** If you need to pass additional parameters to your transformer's constructor +beyond the default ones provided: ```java // Your custom parameter @@ -53,7 +54,7 @@ String databasePath = "/my/path"; DaVinciRecordTransformerFunctionalInterface transformerFunction = ( // Venice-provided parameters: - storeName, storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> + storeName, storeVersion, keySchema, inputValueSchema, outputValueSchema, config) -> new YourTransformer( // Venice-provided parameters storeName, storeVersion, keySchema, inputValueSchema, outputValueSchema, config, @@ -68,26 +69,31 @@ DaVinciRecordTransformerConfig config = new DaVinciRecordTransformerConfig.Build ### Key Concepts #### Version Management + Venice stores have versions. When new data is pushed, Venice creates a **future version** while the **current version** continues serving traffic. Once ready, Venice atomically swaps the future version to become the new current version. #### For Record Transformers: + - Each version gets its own transformer instance -- During a push, you'll have transformers for both current and future versions running in parallel +- During a push, you'll have transformers for both current and future versions running in parallel - Use `onStartVersionIngestion(partitionId, isCurrentVersion)` to initialize resources - Use `onEndVersionIngestion(currentVersion)` to clean up when a version stops serving #### Best Practice: Separate Data by Version -When propagating Venice data to external systems (databases, search indexes, etc.), -**always separate data from different versions into independent storage locations**. -Think of it as maintaining one database table per Venice store version. + +When propagating Venice data to external systems (databases, search indexes, etc.), **always separate data from +different versions into independent storage locations**. Think of it as maintaining one database table per Venice store +version. #### Why Version Separation Matters: + - **Prevents data races**: Multiple versions writing to the same table creates race conditions - **Avoids record leaks**: Old version data won't pollute your current dataset - **Enables clean transitions**: You can atomically switch to new data once ready #### Implementation Strategy: + 1. **Create version-specific storage** (e.g., `user_profiles_v1`, `user_profiles_v2`) 2. **Maintain a pointer to current version** (database views, atomic pointer, etc.) 3. **Switch pointer atomically** when Venice promotes a new current version @@ -105,7 +111,7 @@ synchronized public void onStartVersionIngestion(int partitionId, boolean isCurr if (setUpComplete.get()) { return; } - + // Initialize resources for this version if (!externalDB.containsTable(tableName)) { externalDB.createTable(tableName); @@ -134,28 +140,34 @@ public void onEndVersionIngestion(int currentVersion) { ``` #### Key Behaviors -- **Lazy deserialization**: Keys/values are deserialized lazily to avoid unnecessary CPU/memory overhead if you only need - to inspect some records or parameters + +- **Lazy deserialization**: Keys/values are deserialized lazily to avoid unnecessary CPU/memory overhead if you only + need to inspect some records or parameters - **Startup replay**: Venice replays existing records from disk on startup to rebuild external state -- **Compatibility checks**: Implementation changes are automatically detected and local state is rebuilt to prevent stale data +- **Compatibility checks**: Implementation changes are automatically detected and local state is rebuilt to prevent + stale data ### Featured Implementations -- [VeniceChangelogConsumerDaVinciRecordTransformerImpl](https://github.com/linkedin/venice/blob/main/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImpl.java): + +- [VeniceChangelogConsumerDaVinciRecordTransformerImpl](https://github.com/linkedin/venice/blob/main/clients/da-vinci-client/src/main/java/com/linkedin/davinci/consumer/VeniceChangelogConsumerDaVinciRecordTransformerImpl.java): - The new Venice Change Data Capture (CDC) client was built using the record transformer. - [DuckDBDaVinciRecordTransformer](https://github.com/linkedin/venice/blob/main/integrations/venice-duckdb/src/main/java/com/linkedin/venice/duckdb/DuckDBDaVinciRecordTransformer.java): - Forwards Venice data to DuckDB, allowing you to query your Venice data via SQL. ### Configuration Options -**Required:** -- `setRecordTransformerFunction` - Function that creates your transformer instances - -**Optional:** - - `setKeyClass`: set this if you want to deserialize keys into Avro SpecificRecords. - - `setOutputValueClass` + `setOutputValueSchema`: required together when changing value type/schema or using Avro - SpecificRecords for values. - - `setStoreRecordsInDaVinci` (default: true): persist into Da Vinci’s local disk. - - `setAlwaysBootstrapFromVersionTopic` (default: false): set this to true if `storeRecordsInDaVinci` is false, and - you're storing records in memory without being backed by disk. - - `setRecordTransformationEnabled` (default: true): set to false when returning `UNCHANGED` during `transform`. - - `setRecordMetadataEnabled` (default: false): enable if you need the record metadata in DaVinciRecordTransformerRecordMetadata. \ No newline at end of file +#### Required + +- `setRecordTransformerFunction`: Function that creates your transformer instances + +#### Optional + +- `setKeyClass`: set this if you want to deserialize keys into Avro SpecificRecords. +- `setOutputValueClass` + `setOutputValueSchema`: required together when changing value type/schema or using Avro + SpecificRecords for values. +- `setStoreRecordsInDaVinci` (default: true): persist into Da Vinci’s local disk. +- `setAlwaysBootstrapFromVersionTopic` (default: false): set this to true if `storeRecordsInDaVinci` is false, and + you're storing records in memory without being backed by disk. +- `setRecordTransformationEnabled` (default: true): set to false when returning `UNCHANGED` during `transform`. +- `setRecordMetadataEnabled` (default: false): enable if you need the record metadata in + DaVinciRecordTransformerRecordMetadata. diff --git a/docs/user-guide/read-apis/index.md b/docs/user-guide/read-apis/index.md new file mode 100644 index 00000000000..8f7cf26fc85 --- /dev/null +++ b/docs/user-guide/read-apis/index.md @@ -0,0 +1,7 @@ +# Read APIs + +Venice provides multiple client types: + +- [Da Vinci Client](da-vinci-client.md) - Local caching for ultra-low latency + +Additional clients (Thin Client, Fast Client) documentation coming soon. diff --git a/docs/user_guide/write_api/push_job.md b/docs/user-guide/write-apis/batch-push.md similarity index 71% rename from docs/user_guide/write_api/push_job.md rename to docs/user-guide/write-apis/batch-push.md index 87858ad1354..c76de64099b 100644 --- a/docs/user_guide/write_api/push_job.md +++ b/docs/user-guide/write-apis/batch-push.md @@ -1,86 +1,89 @@ ---- -layout: default -title: Push Job -parent: Write APIs -grand_parent: User Guides -permalink: /docs/user_guide/write_api/push_job ---- # Push Job -The diagram below illustrates the high-level picture of the push job. ([Link to diagram source](https://whimsical.com/venice-docs-diagrams-SvH4RAc9mED9JdAycS5w2v)) +The diagram below illustrates the high-level picture of the push job. +([Link to diagram source](https://whimsical.com/venice-docs-diagrams-SvH4RAc9mED9JdAycS5w2v)) ![Venice Push Job Diagram](https://github.com/user-attachments/assets/bdcefd28-a550-4ab8-9e84-5654efdcb0c0) -The Push Job takes data from a Hadoop grid and writes it to Venice. +The Push Job takes data from a Hadoop grid and writes it to Venice. ## Use Cases + There are two modes the Push Job can run in: - Full Push (default) - Incremental Push ### Full Push + When performing a Full Push, the user takes advantage of the fact that Venice's datasets are versioned. A Full Push -triggers the dynamic creation of a new dataset version, and then loads data into it in the background. The new dataset +triggers the dynamic creation of a new dataset version, and then loads data into it in the background. The new dataset version is called a "future" version as long as data is still loading, and during that time, no online read traffic will -be served from it. When the loading is determined to have successfully completed in a given datacenter, the new dataset -version transitions from "future" to "current", whereas the old dataset version transitions from "current" to "backup". +be served from it. When the loading is determined to have successfully completed in a given datacenter, the new dataset +version transitions from "future" to "current", whereas the old dataset version transitions from "current" to "backup". When a dataset version becomes current, online read traffic gets routed to it. ### Incremental Push -When performing an Incremental Push, no new dataset versions are created, and data gets added into all existing versions -of the dataset. This leverages the same mechanism as Streaming Writes, and requires that the store be configured as + +When performing an Incremental Push, no new dataset versions are created, and data gets added into all existing versions +of the dataset. This leverages the same mechanism as Streaming Writes, and requires that the store be configured as Hybrid. ### Targeted Region Push -Technically, targeted region push is an option of full push _(hence not a new mode)_, but it allows writing data into a + +Technically, targeted region push is an option of full push _(hence not a new mode)_, but it allows writing data into a subset of global regions/data centers, whereas full push writes globally at once. By default, it automatically pushes data to the rest of unspecified regions after the targeted region push is completed. -We are working on to implement more validations in between to ensure the targeted regions are healthy after the first push. -Users may turn off this automation and perform validations and chain it with another full push/targeted region push to -achieve the same effect as full push in terms of data integrity, but the store versions across different regions might be -not the same depending on the exact setup. +We are working on to implement more validations in between to ensure the targeted regions are healthy after the first +push. Users may turn off this automation and perform validations and chain it with another full push/targeted region +push to achieve the same effect as full push in terms of data integrity, but the store versions across different regions +might be not the same depending on the exact setup. ## Usage + The Push Job is designed to require as few configs as possible. The following mandatory configs should be unique to each use case, and set by the user: - `venice.store.name`: The name of the Venice store to push into. -- `input.path`: The HDFS path containing the data to be pushed, populated with one or many Avro files, where each file +- `input.path`: The HDFS path containing the data to be pushed, populated with one or many Avro files, where each file contains a sequence of records, and where each record has a `key` field and a `value` field. -In addition to use case-specific configs, there are also some necessary configs that would typically be the same for all -use cases in a given environment (e.g., one value for production, another value for staging, etc.). The following can +In addition to use case-specific configs, there are also some necessary configs that would typically be the same for all +use cases in a given environment (e.g., one value for production, another value for staging, etc.). The following can therefore be configured globally by the operator, in order to make the Push Job even easier to leverage by users: - `venice.discover.urls`: The URL of the Venice controller. ### Optional Configs + The user may choose to specify the following configs: - `incremental.push`: Whether to run the job in incremental mode. Default: `false` - `key.field`: The name of the key field within the input records. Default: `key` - `value.field`: The name of the value field within the input records. Default: `value` -- `allow.duplicate.key`: Whether to let the Push Job proceed even if it detects that the input contains multiple records +- `allow.duplicate.key`: Whether to let the Push Job proceed even if it detects that the input contains multiple records having the same key but distinct values. If set to `true`, then the Push Job picks one of the values to be written in a non-deterministic fashion. Default: `false` - `extended.schema.validity.check.enabled`: Whether to perform extended schema validation on the input (equivalent to - the `STRICT` mode in avro-util's [SchemaParseConfiguration](https://github.com/linkedin/avro-util/blob/master/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/SchemaParseConfiguration.java)). + the `STRICT` mode in avro-util's + [SchemaParseConfiguration](https://github.com/linkedin/avro-util/blob/master/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/SchemaParseConfiguration.java)). If set to `false`, it becomes equivalent to avro-util's `LOOSE` mode. Default: `true` - `targeted.region.push.enabled`: Whether to perform targeted region push. Default: `false` -- `targeted.region.push.list`: This config takes effect only when targeted region push flag is enabled. - Optionally specify a list of target region(s) to push data into. See full details at +- `targeted.region.push.list`: This config takes effect only when targeted region push flag is enabled. Optionally + specify a list of target region(s) to push data into. See full details at [TARGETED_REGION_PUSH_LIST](https://venicedb.org/javadoc/com/linkedin/venice/hadoop/VenicePushJob.html#TARGETED_REGION_PUSH_LIST). -- `post.validation.consumption`: Whether to perform post validation consumption after targeted region push is finished. - Default: `true`. Set this to `false` if you want to achieve a true single colo push. +- `post.validation.consumption`: Whether to perform post validation consumption after targeted region push is finished. + Default: `true`. Set this to `false` if you want to achieve a true single colo push. The push job also supports using D2 URLs for automated controller service discovery. To use this, the user or operator must specify the following configs: - `multi.region`: `true` if the Venice cluster is deployed in a multi-region mode; `false` otherwise -- `venice.discover.urls`: The D2 URL of the Venice controller. It must be of the form `d2://` -- `d2.zk.hosts.`: The Zookeeper addresses where the components in the specified region are announcing themselves to D2 +- `venice.discover.urls`: The D2 URL of the Venice controller. It must be of the form + `d2://` +- `d2.zk.hosts.`: The Zookeeper addresses where the components in the specified region are announcing + themselves to D2 - If `multi.region` is `true`: - `venice.discover.urls` must use the D2 service name of the parent controller - `parent.controller.region.name` must denote the name of the datacenter where the parent controller is deployed @@ -97,4 +100,4 @@ The user or operator may want to specify the following security-related configs: - `ssl.key.store.property.name` - `ssl.trust.store.property.name` - `ssl.key.store.password.property.name` -- `ssl.key.password.property.name` \ No newline at end of file +- `ssl.key.password.property.name` diff --git a/docs/user-guide/write-apis/index.md b/docs/user-guide/write-apis/index.md new file mode 100644 index 00000000000..81155305578 --- /dev/null +++ b/docs/user-guide/write-apis/index.md @@ -0,0 +1,7 @@ +# Write APIs + +Venice supports multiple write patterns: + +- [Batch Push](batch-push.md) - Full dataset swaps from Hadoop +- [Stream Processor](stream-processor.md) - Real-time updates from Samza +- [Online Producer](online-producer.md) - Direct writes from services diff --git a/docs/user_guide/write_api/online_producer.md b/docs/user-guide/write-apis/online-producer.md similarity index 66% rename from docs/user_guide/write_api/online_producer.md rename to docs/user-guide/write-apis/online-producer.md index a403f0e406c..2b0e726bebd 100644 --- a/docs/user_guide/write_api/online_producer.md +++ b/docs/user-guide/write-apis/online-producer.md @@ -1,42 +1,44 @@ ---- -layout: default -title: Online Producer -parent: Write APIs -grand_parent: User Guides -permalink: /docs/user_guide/write_api/online_producer ---- # Online Producer + The Online Producer enables online applications to write data to a Venice store directly. All writes are still asynchronous, and data is only eventually consistent. However, the APIs guarantee durability of the data if the operation is successful. ## Prerequisites + To use the Online Producer, the store must meet some prerequisites: + 1. It must not have writes disabled 2. It must be a hybrid store 3. It must have a current version. In addition to the store-level prerequisites, the current version must meet the following prerequisites: + 1. It must be configured as hybrid; aka capable of receiving near-line writes 2. It must specify either `ACTIVE_ACTIVE` or `NON_AGGREGATE` data-replication policies 3. It must specify a partitioner that the writer application knows how to use ## API -Detailed Javadocs for the Online Producer API can be accessed [here](https://venicedb.org/javadoc/com/linkedin/venice/producer/VeniceProducer.html). -All of these APIs have at least two versions - one that accepts a logical timestamp and one that doesn't. -1. Logical timestamps (in ms) are what Venice backend will use to resolve conflicts in case multiple writes modify -the same record. An update to Venice could be triggered due to some trigger that can be attributed to a specific -point in time. In such cases, it might be beneficial for applications to mark their updates to Venice with that -timestamp and Venice will persist the record as if it had been received at that point in time - either by applying -the update, dropping the update if a newer update has already been persisted, or applying an update partially only to -fields that have not received an update with a newer timestamp yet. -2. In case the write requests are made without specifying the logical timestamp, then the time at which the message -was produced is used as the logical timestamp during conflict resolution. + +Detailed Javadocs for the Online Producer API can be accessed +[here](https://venicedb.org/javadoc/com/linkedin/venice/producer/VeniceProducer.html). All of these APIs have at least +two versions - one that accepts a logical timestamp and one that doesn't. + +1. Logical timestamps (in ms) are what Venice backend will use to resolve conflicts in case multiple writes modify the + same record. An update to Venice could be triggered due to some trigger that can be attributed to a specific point in + time. In such cases, it might be beneficial for applications to mark their updates to Venice with that timestamp and + Venice will persist the record as if it had been received at that point in time - either by applying the update, + dropping the update if a newer update has already been persisted, or applying an update partially only to fields that + have not received an update with a newer timestamp yet. +2. In case the write requests are made without specifying the logical timestamp, then the time at which the message was + produced is used as the logical timestamp during conflict resolution. ## Usage + To create an instance of the producer, `OnlineProducerFactory` should be used since the interface for `OnlineVeniceProducer` is not yet considered stable and can introduce backward incompatible changes. -``` + +```java String storeName = ""; ClientConfig clientConfig = ClientConfig.defaultGenericClientConfig(storeName) .setVeniceURL("http://router.host:7777") @@ -55,15 +57,18 @@ producer.asyncUpdate(key, builder -> { ``` ## Optional Configs + The online Venice producer can be configured by specifying the following optional configs: To specify the number of threads dedicated for the online Venice producer. This also controls the number of concurrent write operations in each producer: + ``` client.producer.thread.num = 10 ``` To specify the interval at which the client will refresh its schema cache: + ``` client.producer.schema.refresh.interval.seconds = 300 ``` diff --git a/docs/user-guide/write-apis/stream-processor.md b/docs/user-guide/write-apis/stream-processor.md new file mode 100644 index 00000000000..165c6772a05 --- /dev/null +++ b/docs/user-guide/write-apis/stream-processor.md @@ -0,0 +1,15 @@ +# Stream Processor + +Data can be produced to Venice in a nearline fashion, from stream processors. The best supported stream processor is +Apache Samza though we intend to add first-class support for other stream processors in the future. The difference +between using a stream processor library and the [Online Producer](online-producer.md) library is that a stream +processor has well-defined semantics around when to ensure that produced data is flushed and a built-in mechanism to +checkpoint its progress relative to its consumption progress in upstream data sources, whereas the online producer +library is a lower-level building block which leaves these reliability details up to the user. + +For Apache Samza, the integration point is done at the level of the +[VeniceSystemProducer](https://github.com/linkedin/venice/blob/main/integrations/venice-samza/src/main/java/com/linkedin/venice/samza/VeniceSystemProducer.java) +and +[VeniceSystemFactory](https://github.com/linkedin/venice/blob/main/integrations/venice-samza/src/main/java/com/linkedin/venice/samza/VeniceSystemFactory.java). + +More details to come. diff --git a/docs/user_guide/design_patterns/index.md b/docs/user_guide/design_patterns/index.md deleted file mode 100644 index dffff2b1f32..00000000000 --- a/docs/user_guide/design_patterns/index.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -layout: default -title: Design Patterns -parent: User Guides -permalink: /docs/user_guide/design_patterns ---- - -# Design Patterns - -Venice is a flexible toolkit that can be used in a variety of ways. Although some ways of using Venice may be easier -while others are more sophisticated, the intent is not to be opinionated (although higher level layers on top of Venice -may choose to be opinionated). - -This section documents various design patterns that can be implemented by Venice users. The goal is to explain how each -pattern works, what semantics it provides, and ultimately help the architect pick the right tool for the job. \ No newline at end of file diff --git a/docs/user_guide/read_api/read_api.md b/docs/user_guide/read_api/read_api.md deleted file mode 100644 index 21ebe21b208..00000000000 --- a/docs/user_guide/read_api/read_api.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -layout: default -title: Read APIs -parent: User Guides -has_children: true -permalink: /docs/user_guide/read_api ---- -# Read APIs - -This section includes guides for reading data from Venice. \ No newline at end of file diff --git a/docs/user_guide/user_guide.md b/docs/user_guide/user_guide.md deleted file mode 100644 index 271019d8314..00000000000 --- a/docs/user_guide/user_guide.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: default -title: User Guides -has_children: true -permalink: /docs/user_guide ---- -# User Guide - -This folder includes guides for Venice users, including the supported read and write APIs. - -![API Overview](../assets/images/api_overview.drawio.svg) \ No newline at end of file diff --git a/docs/user_guide/write_api/stream_processor.md b/docs/user_guide/write_api/stream_processor.md deleted file mode 100644 index 91033ef1a41..00000000000 --- a/docs/user_guide/write_api/stream_processor.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -layout: default -title: Stream Processor -parent: Write APIs -grand_parent: User Guides -permalink: /docs/user_guide/write_api/stream_processor ---- -# Stream Processor - -Data can be produced to Venice in a nearline fashion, from stream processors. The best supported stream processor is -Apache Samza though we intend to add first-class support for other stream processors in the future. The difference -between using a stream processor library and the [Online Producer](./online_producer.md) library is that a stream -processor has well-defined semantics around when to ensure that produced data is flushed and a built-in mechanism to -checkpoint its progress relative to its consumption progress in upstream data sources, whereas the online producer -library is a lower-level building block which leaves these reliability details up to the user. - -For Apache Samza, the integration point is done at the level of the [VeniceSystemProducer](https://github.com/linkedin/venice/blob/main/integrations/venice-samza/src/main/java/com/linkedin/venice/samza/VeniceSystemProducer.java) -and [VeniceSystemFactory](https://github.com/linkedin/venice/blob/main/integrations/venice-samza/src/main/java/com/linkedin/venice/samza/VeniceSystemFactory.java). - -More details to come. \ No newline at end of file diff --git a/docs/user_guide/write_api/write_api.md b/docs/user_guide/write_api/write_api.md deleted file mode 100644 index c6aa009ac77..00000000000 --- a/docs/user_guide/write_api/write_api.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -layout: default -title: Write APIs -parent: User Guides -has_children: true -permalink: /docs/user_guide/write_api ---- -# Write APIs - -This section includes guides for writing data to Venice. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000000..e4fa37c1590 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,156 @@ +site_name: Venice +site_url: https://venicedb.org +site_description: Derived Data Platform for Planet-Scale Workloads +site_author: Venice +repo_url: https://github.com/linkedin/venice +repo_name: linkedin/venice +edit_uri: edit/main/docs/ + +theme: + name: material + language: en + logo: assets/style/HEAD_white_RGB.svg + favicon: assets/style/favicon.ico + + palette: + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: white + toggle: + icon: material/weather-sunny + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: black + accent: black + toggle: + icon: material/weather-night + name: Switch to dark mode + + features: + - navigation.instant + - navigation.instant.progress + - navigation.tracking + - navigation.tabs + - navigation.tabs.sticky + - navigation.sections + - navigation.path + - navigation.indexes + - navigation.top + - navigation.footer + - search.suggest + - search.highlight + - content.code.copy + - content.action.edit + - toc.follow + +extra_css: + - assets/style/extra.css + +markdown_extensions: + - abbr + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - tables + - mdx_truly_sane_lists + - toc: + permalink: "#" + toc_depth: 3 + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + +plugins: + - search + - git-revision-date-localized: + enable_creation_date: true + - minify: + minify_html: true + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/linkedin/venice + - icon: fontawesome/brands/slack + link: http://slack.venicedb.org + - icon: fontawesome/brands/linkedin + link: https://www.linkedin.com/company/venicedb/ + - icon: fontawesome/brands/x-twitter + link: https://x.com/VeniceDataBase + - icon: fontawesome/brands/youtube + link: https://youtube.com/@venicedb + - icon: fontawesome/solid/rss + link: https://blog.venicedb.org + +nav: + - Home: README.md + + - Getting Started: + - getting-started/index.md + - Learn Venice: + - Architecture Overview: getting-started/learn-venice/architecture-overview.md + - Merging Batch & Real-Time Data: getting-started/learn-venice/merging-batch-and-rt-data.md + - Deployments: + - Single Datacenter: getting-started/deployments/quickstart-single-dc.md + - Multi Datacenter: getting-started/deployments/quickstart-multi-dc.md + + - User Guide: + - user-guide/index.md + - Write APIs: + - user-guide/write-apis/index.md + - Batch Push: user-guide/write-apis/batch-push.md + - Stream Processor: user-guide/write-apis/stream-processor.md + - Online Producer: user-guide/write-apis/online-producer.md + - Read APIs: + - user-guide/read-apis/index.md + - Da Vinci Client: user-guide/read-apis/da-vinci-client.md + + - Operations: + - operations/index.md + - Data Management: + - Repush: operations/data-management/repush.md + - TTL: operations/data-management/ttl.md + - System Stores: operations/data-management/system-stores.md + - Advanced: + - P2P Bootstrapping: operations/advanced/p2p-bootstrapping.md + - Data Integrity: operations/advanced/data-integrity.md + + - Contributing: + - contributing/index.md + - Code of Conduct: contributing/code-of-conduct.md + - Contributing Guide: contributing/contributing.md + - Security: contributing/security.md + - Development: + - Workspace Setup: contributing/development/workspace-setup.md + - Dev Workflow: contributing/development/dev-workflow.md + - Testing: contributing/development/testing.md + - Style Guide: contributing/development/style-guide.md + - Architecture: + - Project Navigation: contributing/architecture/navigation.md + - Write Path: contributing/architecture/write-path.md + - Java Internals: contributing/architecture/java-internals.md + - Router API: contributing/architecture/router-api.md + - Documentation: + - Writing Docs: contributing/documentation/writing-docs.md + - Design Docs: contributing/documentation/design-docs.md + - Proposals: + - contributing/proposals/index.md + - VIP Template: contributing/proposals/vip-template.md + - VIP-1: contributing/proposals/vip-1.md + - VIP-2: contributing/proposals/vip-2.md + - VIP-3: contributing/proposals/vip-3.md + - VIP-4: contributing/proposals/vip-4.md + - VIP-5: contributing/proposals/vip-5.md + - VIP-6: contributing/proposals/vip-6.md + + - Resources: + - Learn More: resources/learn-more.md