Merge branch 'master' into minaasham/upgrade-replace-avro-python3

apache · Jan 21, 2025 · 8d2543b · 8d2543b
2 parents 2bddaa7 + 9ba5afc
commit 8d2543b
Show file tree

Hide file tree

Showing 414 changed files with 15,049 additions and 1,823 deletions.
diff --git a/.github/workflows/nightly-trigger.yml b/.github/workflows/nightly-trigger.yml
@@ -33,7 +33,6 @@ jobs:
           - master
           - release-1.20
           - release-1.19
-          - release-1.18
     runs-on: ubuntu-latest
     steps:
       - name: Trigger Workflow

diff --git a/docs/config.toml b/docs/config.toml
@@ -34,11 +34,11 @@ pygmentsUseClasses = true
   # we change the version for the complete docs when forking of a release branch
   # etc.
   # The full version string as referenced in Maven (e.g. 1.2.1)
-  Version = "2.0-SNAPSHOT"
+  Version = "2.1-SNAPSHOT"
 
   # For stable releases, leave the bugfix version out (e.g. 1.2). For snapshot
   # release this should be the same as the regular version
-  VersionTitle = "2.0-SNAPSHOT"
+  VersionTitle = "2.1-SNAPSHOT"
 
   # The branch for this version of Apache Flink
   Branch = "master"

diff --git a/docs/content.zh/docs/deployment/cli.md b/docs/content.zh/docs/deployment/cli.md
@@ -576,7 +576,7 @@ related options. Here's an overview of all the Python related options for the ac
             <td>
                 Specify the path of the python interpreter used to execute the python UDF worker
                 (e.g.: --pyExecutable /usr/local/bin/python3).
-                The python UDF worker depends on Python 3.8+, Apache Beam (version == 2.43.0),
+                The python UDF worker depends on Python 3.8+, Apache Beam (version >= 2.54.0, <= 2.61.0),
                 Pip (version >= 20.3) and SetupTools (version >= 37.0.0).
                 Please ensure that the specified environment meets the above requirements.
             </td>

diff --git a/docs/content.zh/docs/dev/table/sqlClient.md b/docs/content.zh/docs/dev/table/sqlClient.md
@@ -322,7 +322,7 @@ Mode "embedded" (default) submits Flink jobs from the local machine.
                                                 /usr/local/bin/python3). The
                                                 python UDF worker depends on
                                                 Python 3.8+, Apache Beam
-                                                (version == 2.43.0), Pip
+                                                (version >= 2.54.0, <= 2.61.0), Pip
                                                 (version >= 20.3) and SetupTools
                                                 (version >= 37.0.0). Please
                                                 ensure that the specified

diff --git a/docs/content.zh/docs/ops/metrics.md b/docs/content.zh/docs/ops/metrics.md
@@ -1545,31 +1545,33 @@ Certain RocksDB native metrics are available but disabled by default, you can fi
 <table class="table table-bordered">
   <thead>
     <tr>
-      <th class="text-left" style="width: 18%">Scope</th>
-      <th class="text-left" style="width: 26%">Metrics</th>
-      <th class="text-left" style="width: 48%">Description</th>
-      <th class="text-left" style="width: 8%">Type</th>
+      <th class="text-left" style="width: 15%">Scope</th>
+      <th class="text-left" style="width: 15%">Infix</th>
+      <th class="text-left" style="width: 15%">Metrics</th>
+      <th class="text-left" style="width: 50%">Description</th>
+      <th class="text-left" style="width: 5%">Type</th>
     </tr>
   </thead>
   <tbody>
     <tr>
       <th rowspan="4"><strong>Task/Operator</strong></th>
-      <td>forst.fileCache.hit</td>
+      <td rowspan="4">forst.fileCache</td>
+      <td>hit</td>
       <td>The hit count of ForSt state backend cache.</td>
       <td>Counter</td>
     </tr>
     <tr>
-      <td>forst.fileCache.miss</td>
+      <td>miss</td>
       <td>The miss count of ForSt state backend cache.</td>
       <td>Counter</td>
     </tr>
     <tr>
-      <td>forst.fileCache.usedBytes</td>
+      <td>usedBytes</td>
       <td>The bytes cached in ForSt state backend cache.</td>
       <td>Gauge</td>
     </tr>
     <tr>
-      <td>forst.fileCache.remainingBytes</td>
+      <td>remainingBytes</td>
       <td>The remaining space in the volume for the configured cache. Only available when 'state.backend.forst.cache.reserve-size' is set above 0. </td>
       <td>Gauge</td>
     </tr>
@@ -2280,6 +2282,44 @@ logged by `SystemResourcesMetricsInitializer` during the startup.
   </tbody>
 </table>
 
+### Async State Processing
+
+<table class="table table-bordered">
+  <thead>
+    <tr>
+      <th class="text-left" style="width: 15%">Scope</th>
+      <th class="text-left" style="width: 10%">Infix</th>
+      <th class="text-left" style="width: 20%">Metrics</th>
+      <th class="text-left" style="width: 50%">Description</th>
+      <th class="text-left" style="width: 5%">Type</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th rowspan="4"><strong>Operator</strong></th>
+      <td rowspan="4">asyncStateProcessing</td>
+      <td>numInFlightRecords</td>
+      <td>The number of in-flight records in the async execution controller's buffers.</td>
+      <td>Gauge</td>
+    </tr>
+    <tr>
+      <td>activeBufferSize</td>
+      <td>The number of records which are pending to be processed.</td>
+      <td>Gauge</td>
+    </tr>
+    <tr>
+      <td>blockingBufferSize</td>
+      <td>The number of records which are blocked by the ongoing records.</td>
+      <td>Gauge</td>
+    </tr>
+    <tr>
+      <td>numBlockingKeys</td>
+      <td>The number of different keys are blocked in async execution controller.</td>
+      <td>Gauge</td>
+    </tr>
+  </tbody>
+</table>
+
 ## End-to-End latency tracking
 
 Flink allows to track the latency of records travelling through the system. This feature is disabled by default.

diff --git a/docs/content/docs/deployment/cli.md b/docs/content/docs/deployment/cli.md
@@ -574,7 +574,7 @@ related options. Here's an overview of all the Python related options for the ac
             <td>
                 Specify the path of the python interpreter used to execute the python UDF worker
                 (e.g.: --pyExecutable /usr/local/bin/python3).
-                The python UDF worker depends on Python 3.8+, Apache Beam (version == 2.43.0),
+                The python UDF worker depends on Python 3.8+, Apache Beam (version >= 2.54.0,<= 2.61.0),
                 Pip (version >= 20.3) and SetupTools (version >= 37.0.0).
                 Please ensure that the specified environment meets the above requirements.
             </td>

diff --git a/docs/content/docs/dev/table/sqlClient.md b/docs/content/docs/dev/table/sqlClient.md
@@ -260,7 +260,7 @@ Mode "embedded" (default) submits Flink jobs from the local machine.
                                                 /usr/local/bin/python3). The
                                                 python UDF worker depends on
                                                 Python 3.8+, Apache Beam
-                                                (version == 2.43.0), Pip
+                                                (version >= 2.54.0, <= 2.61.0), Pip
                                                 (version >= 20.3) and SetupTools
                                                 (version >= 37.0.0). Please
                                                 ensure that the specified

diff --git a/docs/content/docs/ops/metrics.md b/docs/content/docs/ops/metrics.md
@@ -1535,31 +1535,33 @@ Certain RocksDB native metrics are available but disabled by default, you can fi
 <table class="table table-bordered">
   <thead>
     <tr>
-      <th class="text-left" style="width: 18%">Scope</th>
-      <th class="text-left" style="width: 26%">Metrics</th>
-      <th class="text-left" style="width: 48%">Description</th>
-      <th class="text-left" style="width: 8%">Type</th>
+      <th class="text-left" style="width: 15%">Scope</th>
+      <th class="text-left" style="width: 15%">Infix</th>
+      <th class="text-left" style="width: 15%">Metrics</th>
+      <th class="text-left" style="width: 50%">Description</th>
+      <th class="text-left" style="width: 5%">Type</th>
     </tr>
   </thead>
   <tbody>
     <tr>
       <th rowspan="4"><strong>Task/Operator</strong></th>
-      <td>forst.fileCache.hit</td>
+      <td rowspan="4">forst.fileCache</td>
+      <td>hit</td>
       <td>The hit count of ForSt state backend cache.</td>
       <td>Counter</td>
     </tr>
     <tr>
-      <td>forst.fileCache.miss</td>
+      <td>miss</td>
       <td>The miss count of ForSt state backend cache.</td>
       <td>Counter</td>
     </tr>
     <tr>
-      <td>forst.fileCache.usedBytes</td>
+      <td>usedBytes</td>
       <td>The bytes cached in ForSt state backend cache.</td>
       <td>Gauge</td>
     </tr>
     <tr>
-      <td>forst.fileCache.remainingBytes</td>
+      <td>remainingBytes</td>
       <td>The remaining space in the volume for the configured cache. Only available when 'state.backend.forst.cache.reserve-size' is set above 0. </td>
       <td>Gauge</td>
     </tr>
@@ -2230,6 +2232,44 @@ Metrics below can be used to measure the effectiveness of speculative execution.
   </tbody>
 </table>
 
+### Async State Processing
+
+<table class="table table-bordered">
+  <thead>
+    <tr>
+      <th class="text-left" style="width: 15%">Scope</th>
+      <th class="text-left" style="width: 10%">Infix</th>
+      <th class="text-left" style="width: 20%">Metrics</th>
+      <th class="text-left" style="width: 50%">Description</th>
+      <th class="text-left" style="width: 5%">Type</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th rowspan="4"><strong>Operator</strong></th>
+      <td rowspan="4">asyncStateProcessing</td>
+      <td>numInFlightRecords</td>
+      <td>The number of in-flight records in the async execution controller's buffers.</td>
+      <td>Gauge</td>
+    </tr>
+    <tr>
+      <td>activeBufferSize</td>
+      <td>The number of records which are pending to be processed.</td>
+      <td>Gauge</td>
+    </tr>
+    <tr>
+      <td>blockingBufferSize</td>
+      <td>The number of records which are blocked by the ongoing records.</td>
+      <td>Gauge</td>
+    </tr>
+    <tr>
+      <td>numBlockingKeys</td>
+      <td>The number of different keys are blocked in async execution controller.</td>
+      <td>Gauge</td>
+    </tr>
+  </tbody>
+</table>
+
 ## End-to-End latency tracking
 
 Flink allows to track the latency of records travelling through the system. This feature is disabled by default.

diff --git a/docs/layouts/shortcodes/generated/python_configuration.html b/docs/layouts/shortcodes/generated/python_configuration.html
@@ -24,7 +24,7 @@
             <td><h5>python.executable</h5></td>
             <td style="word-wrap: break-word;">"python"</td>
             <td>String</td>
-            <td>Specify the path of the python interpreter used to execute the python UDF worker. The python UDF worker depends on Python 3.8+, Apache Beam (version == 2.43.0), Pip (version &gt;= 20.3) and SetupTools (version &gt;= 37.0.0). Please ensure that the specified environment meets the above requirements. The option is equivalent to the command line option "-pyexec".</td>
+            <td>Specify the path of the python interpreter used to execute the python UDF worker. The python UDF worker depends on Python 3.8+, Apache Beam (version >= 2.54.0, <= 2.61.0), Pip (version &gt;= 20.3) and SetupTools (version &gt;= 37.0.0). Please ensure that the specified environment meets the above requirements. The option is equivalent to the command line option "-pyexec".</td>
         </tr>
         <tr>
             <td><h5>python.execution-mode</h5></td>

diff --git a/docs/setup_hugo.sh b/docs/setup_hugo.sh
@@ -18,8 +18,19 @@
 ################################################################################
 
 # setup hugo
-HUGO_REPO=https://github.com/gohugoio/hugo/releases/download/v0.110.0/hugo_extended_0.110.0_Linux-64bit.tar.gz
-HUGO_ARTIFACT=hugo_extended_0.110.0_Linux-64bit.tar.gz
+
+# Detect Operating System
+OS="Linux"
+[[ "$OSTYPE" == "darwin"* ]] && OS="Mac"
+
+# Setup Hugo based on OS
+if [ "$OS" = "Mac" ]; then
+    HUGO_ARTIFACT="hugo_extended_0.110.0_darwin-universal.tar.gz"
+else
+    HUGO_ARTIFACT="hugo_extended_0.110.0_Linux-64bit.tar.gz"
+fi
+
+HUGO_REPO="https://github.com/gohugoio/hugo/releases/download/v0.110.0/${HUGO_ARTIFACT}"
 if ! curl --fail -OL $HUGO_REPO ; then
 	echo "Failed to download Hugo binary"
 	exit 1

diff --git a/flink-annotations/pom.xml b/flink-annotations/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-annotations</artifactId>

diff --git a/flink-architecture-tests/flink-architecture-tests-base/pom.xml b/flink-architecture-tests/flink-architecture-tests-base/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-architecture-tests</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-architecture-tests-base</artifactId>

diff --git a/flink-architecture-tests/flink-architecture-tests-production/pom.xml b/flink-architecture-tests/flink-architecture-tests-production/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-architecture-tests</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-architecture-tests-production</artifactId>

diff --git a/flink-architecture-tests/flink-architecture-tests-test/pom.xml b/flink-architecture-tests/flink-architecture-tests-test/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-architecture-tests</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-architecture-tests-test</artifactId>

diff --git a/flink-architecture-tests/pom.xml b/flink-architecture-tests/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-architecture-tests</artifactId>

diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-clients</artifactId>

diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java
@@ -275,7 +275,7 @@ public class CliFrontendParser {
                     true,
                     "Specify the path of the python interpreter used to execute the python UDF worker "
                             + "(e.g.: --pyExecutable /usr/local/bin/python3). "
-                            + "The python UDF worker depends on Python 3.8+, Apache Beam (version == 2.43.0), "
+                            + "The python UDF worker depends on Python 3.8+, Apache Beam (version >= 2.54.0, <= 2.61.0), "
                             + "Pip (version >= 20.3) and SetupTools (version >= 37.0.0). "
                             + "Please ensure that the specified environment meets the above requirements.");
 

diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/StandaloneClusterId.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/StandaloneClusterId.java
@@ -27,4 +27,9 @@ private StandaloneClusterId() {}
     public static StandaloneClusterId getInstance() {
         return INSTANCE;
     }
+
+    @Override
+    public String toString() {
+        return "StandaloneClusterId";
+    }
 }
diff --git a/flink-connectors/flink-connector-base/pom.xml b/flink-connectors/flink-connector-base/pom.xml
@@ -25,7 +25,7 @@
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-connector-base</artifactId>

diff --git a/flink-connectors/flink-connector-datagen-test/pom.xml b/flink-connectors/flink-connector-datagen-test/pom.xml
@@ -25,7 +25,7 @@
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-connector-datagen-tests</artifactId>

diff --git a/flink-connectors/flink-connector-datagen/pom.xml b/flink-connectors/flink-connector-datagen/pom.xml
@@ -25,7 +25,7 @@
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-connector-datagen</artifactId>

diff --git a/flink-connectors/flink-connector-files/pom.xml b/flink-connectors/flink-connector-files/pom.xml
@@ -26,7 +26,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-connector-files</artifactId>

diff --git a/flink-connectors/flink-connector-hive/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-connector-hive/src/main/resources/META-INF/NOTICE
@@ -6,9 +6,9 @@ The Apache Software Foundation (http://www.apache.org/).
 
 This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt)
 
-- org.apache.parquet:parquet-hadoop:1.13.1
-- org.apache.parquet:parquet-column:1.13.1
-- org.apache.parquet:parquet-common:1.13.1
-- org.apache.parquet:parquet-encoding:1.13.1
-- org.apache.parquet:parquet-format-structures:1.13.1
-- org.apache.parquet:parquet-jackson:1.13.1
+- org.apache.parquet:parquet-hadoop:1.14.4
+- org.apache.parquet:parquet-column:1.14.4
+- org.apache.parquet:parquet-common:1.14.4
+- org.apache.parquet:parquet-encoding:1.14.4
+- org.apache.parquet:parquet-format-structures:1.14.4
+- org.apache.parquet:parquet-jackson:1.14.4
diff --git a/flink-connectors/flink-file-sink-common/pom.xml b/flink-connectors/flink-file-sink-common/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-connectors</artifactId>
-		<version>2.0-SNAPSHOT</version>
+		<version>2.1-SNAPSHOT</version>
 	</parent>
 
 	<artifactId>flink-file-sink-common</artifactId>