KAFKA-18052: Decouple the dependency of feature stable version to the metadata version

server-common/src/main/java/org/apache/kafka/server/common/Features.java

@@ -41,10 +41,10 @@ public enum Features {
      *
      * See {@link TestFeatureVersion} as an example. See {@link FeatureVersion} when implementing a new feature.
      */
-    TEST_VERSION("test.feature.version", TestFeatureVersion.values()),
-    KRAFT_VERSION("kraft.version", KRaftVersion.values()),
-    TRANSACTION_VERSION("transaction.version", TransactionVersion.values()),
-    GROUP_VERSION("group.version", GroupVersion.values());
+    TEST_VERSION("test.feature.version", TestFeatureVersion.values(), TestFeatureVersion.LATEST_PRODUCTION),
+    KRAFT_VERSION("kraft.version", KRaftVersion.values(), KRaftVersion.LATEST_PRODUCTION),
+    TRANSACTION_VERSION("transaction.version", TransactionVersion.values(), TransactionVersion.LATEST_PRODUCTION),
+    GROUP_VERSION("group.version", GroupVersion.values(), GroupVersion.LATEST_PRODUCTION);
 
     public static final Features[] FEATURES;
     public static final List<Features> PRODUCTION_FEATURES;
@@ -53,10 +53,17 @@ public enum Features {
     private final String name;
     private final FeatureVersion[] featureVersions;
 
+    // The latest production version of the feature, owned and updated by the feature owner
+    // in the respective feature definition. The value should not be smaller than the default
+    // value calculated with {@link #defaultValue(MetadataVersion)}.
+    private final FeatureVersion latestProduction;
+
     Features(String name,
-             FeatureVersion[] featureVersions) {
+             FeatureVersion[] featureVersions,
+             FeatureVersion latestProduction) {
         this.name = name;
         this.featureVersions = featureVersions;
+        this.latestProduction = latestProduction;
     }
 
     static {
@@ -67,6 +74,10 @@ public enum Features {
                 !feature.name.equals(TEST_VERSION.featureName())).collect(Collectors.toList());
         PRODUCTION_FEATURE_NAMES = PRODUCTION_FEATURES.stream().map(feature ->
                 feature.name).collect(Collectors.toList());
+
+        for (Features feature : enumValues) {
+            validateDefaultValueAndLatestProductionValue(feature);
+        }
     }
 
     public String featureName() {
@@ -78,7 +89,7 @@ public FeatureVersion[] featureVersions() {
     }
 
     public short latestProduction() {
-        return defaultValue(MetadataVersion.LATEST_PRODUCTION);
+        return latestProduction.featureLevel();
     }
 
     public short minimumProduction() {
@@ -142,25 +153,29 @@ public static void validateVersion(FeatureVersion feature, Map<String, Short> fe
     }
 
     /**
-     * A method to return the default (latest production) level of a feature based on the metadata version provided.
+     * A method to return the default (latest production) version of a feature based on the metadata version provided.
      *
      * Every time a new feature is added, it should create a mapping from metadata version to feature version
-     * with {@link FeatureVersion#bootstrapMetadataVersion()}. When the feature version is production ready, the metadata
-     * version should be made production ready as well.
+     * with {@link FeatureVersion#bootstrapMetadataVersion()}. The feature version should be marked as production ready
+     * before the metadata version is made production ready.
      *
      * @param metadataVersion the metadata version we want to use to set the default.
-     * @return the default version level given the feature and provided metadata version
+     * @return the default version given the feature and provided metadata version
      */
-    public short defaultValue(MetadataVersion metadataVersion) {
-        short level = 0;
+    public FeatureVersion defaultVersion(MetadataVersion metadataVersion) {
+        FeatureVersion version = featureVersions[0];
         for (Iterator<FeatureVersion> it = Arrays.stream(featureVersions).iterator(); it.hasNext(); ) {
             FeatureVersion feature = it.next();
             if (feature.bootstrapMetadataVersion().isLessThan(metadataVersion) || feature.bootstrapMetadataVersion().equals(metadataVersion))
-                level = feature.featureLevel();
+                version = feature;
             else
-                return level;
+                return version;
         }
-        return level;
+        return version;
+    }
+
+    public short defaultValue(MetadataVersion metadataVersion) {
+        return defaultVersion(metadataVersion).featureLevel();
     }
 
     public static Features featureFromName(String featureName) {
@@ -177,4 +192,51 @@ public static Features featureFromName(String featureName) {
     public static Map<String, Short> featureImplsToMap(List<FeatureVersion> features) {
         return features.stream().collect(Collectors.toMap(FeatureVersion::featureName, FeatureVersion::featureLevel));
     }
+
+    public boolean isProductionReady(short featureVersion) {
+        return featureVersion <= latestProduction();
+    }
+
+    /**
+     * The method ensures that the following statements are met:
+     * 1. The latest production value >= the default value.
+     * 2. The dependencies of the latest production value <= their latest production values.
+     * 3. The dependencies of the default value <= their default values.
+     */
+    public static void validateDefaultValueAndLatestProductionValue(Features feature) {
+        FeatureVersion defaultVersion = feature.defaultVersion(MetadataVersion.LATEST_PRODUCTION);
+        if (feature.latestProduction() < defaultVersion.featureLevel()) {
+            throw new IllegalArgumentException("The latest production value should be no smaller than the default value.");
+        }
+
+        for (Map.Entry<String, Short> dependency: feature.latestProduction.dependencies().entrySet()) {
+            Features dependencyFeature;
+            try {
+                dependencyFeature = featureFromName(dependency.getKey());
+            } catch (IllegalArgumentException e) {
+                // There might be feature depending on MetadataVersion which is not a feature,
+                // so we just skip checking it.
+                continue;
+            }
+
+            if (!dependencyFeature.isProductionReady(dependency.getValue())) {
+                throw new IllegalArgumentException("The latest production value has a dependency that is not production ready.");
+            }
+        }
+
+        for (Map.Entry<String, Short> dependency: defaultVersion.dependencies().entrySet()) {
+            Features dependencyFeature;
+            try {
+                dependencyFeature = featureFromName(dependency.getKey());
+            } catch (IllegalArgumentException e) {
+                // There might be feature depending on MetadataVersion which is not a feature,
+                // so we just skip checking it.
+                continue;
+            }
+
+            if (dependency.getValue() > dependencyFeature.defaultValue(MetadataVersion.LATEST_PRODUCTION)) {
+                throw new IllegalArgumentException("The default value has a dependency that is ahead of its default value.");
+            }
+        }
+    }
 }

server-common/src/main/java/org/apache/kafka/server/common/GroupVersion.java

@@ -29,6 +29,8 @@ public enum GroupVersion implements FeatureVersion {
 
     public static final String FEATURE_NAME = "group.version";
 
+    public static final GroupVersion LATEST_PRODUCTION = GV_1;
+
     private final short featureLevel;
     private final MetadataVersion bootstrapMetadataVersion;
     private final Map<String, Short> dependencies;

server-common/src/main/java/org/apache/kafka/server/common/KRaftVersion.java

@@ -28,6 +28,8 @@ public enum KRaftVersion implements FeatureVersion {
 
     public static final String FEATURE_NAME = "kraft.version";
 
+    public static final KRaftVersion LATEST_PRODUCTION = KRAFT_VERSION_1;
+
     private final short featureLevel;
     private final MetadataVersion bootstrapMetadataVersion;
 

server-common/src/main/java/org/apache/kafka/server/common/TestFeatureVersion.java

@@ -32,6 +32,9 @@ public enum TestFeatureVersion implements FeatureVersion {
 
     public static final String FEATURE_NAME = "test.feature.version";
 
+    public static final TestFeatureVersion LATEST_PRODUCTION =
+        MetadataVersion.latestProduction() == MetadataVersion.latestTesting() ? TEST_2 : TEST_1;
+
     TestFeatureVersion(int featureLevel, MetadataVersion metadataVersionMapping, Map<String, Short> dependencies) {
         this.featureLevel = (short) featureLevel;
         this.metadataVersionMapping = metadataVersionMapping;

server-common/src/main/java/org/apache/kafka/server/common/TransactionVersion.java

@@ -30,6 +30,8 @@ public enum TransactionVersion implements FeatureVersion {
 
     public static final String FEATURE_NAME = "transaction.version";
 
+    public static final TransactionVersion LATEST_PRODUCTION = TV_0;
+
     private final short featureLevel;
     private final MetadataVersion bootstrapMetadataVersion;
     private final Map<String, Short> dependencies;

server-common/src/test/java/org/apache/kafka/server/common/FeaturesTest.java

@@ -115,8 +115,8 @@ public void testDefaultValueAllFeatures(Features feature) {
 
     @ParameterizedTest
     @EnumSource(Features.class)
-    public void testLatestProductionMapsToLatestMetadataVersion(Features features) {
-        assertEquals(features.latestProduction(), features.defaultValue(MetadataVersion.LATEST_PRODUCTION));
+    public void testLatestProductionIsNotBehindLatestMetadataVersion(Features features) {
+        assertTrue(features.latestProduction() >= features.defaultValue(MetadataVersion.latestProduction()));
     }
 
     @ParameterizedTest