diff --git a/.gitignore b/.gitignore index ff2c78719c82c..7966b633e0e21 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ scalastyle-output.xml .metals .bloop .cursor +.claude .metadata .settings .project diff --git a/flink-python/pyflink/table/tests/test_environment_settings_completeness.py b/flink-python/pyflink/table/tests/test_environment_settings_completeness.py index 88da848c8ef21..5ce757f24721d 100644 --- a/flink-python/pyflink/table/tests/test_environment_settings_completeness.py +++ b/flink-python/pyflink/table/tests/test_environment_settings_completeness.py @@ -38,7 +38,7 @@ def java_class(cls): def excluded_methods(cls): # internal interfaces, no need to expose to users. return {'getPlanner', 'getExecutor', 'getUserClassLoader', 'getCatalogStore', - 'toConfiguration', 'fromConfiguration', 'getSqlFactory'} + 'getSecretStore', 'toConfiguration', 'fromConfiguration', 'getSqlFactory'} class EnvironmentSettingsBuilderCompletenessTests(PythonAPICompletenessTestCase, PyFlinkTestCase): @@ -59,7 +59,7 @@ def java_class(cls): def excluded_methods(cls): # internal interfaces, no need to expose to users. # withSqlFactory - needs to be implemented - return {'withClassLoader', 'withCatalogStore', 'withSqlFactory'} + return {'withClassLoader', 'withCatalogStore', 'withSecretStore', 'withSqlFactory'} if __name__ == '__main__': import unittest diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java index 12bfc2d8e5a87..b5463d44267fe 100644 --- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java @@ -31,6 +31,7 @@ import org.apache.flink.table.catalog.CatalogStoreHolder; import org.apache.flink.table.catalog.FunctionCatalog; import org.apache.flink.table.catalog.GenericInMemoryCatalog; +import org.apache.flink.table.factories.ApiFactoryUtil; import org.apache.flink.table.factories.CatalogStoreFactory; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.table.factories.TableFactoryUtil; @@ -390,9 +391,9 @@ private static CatalogManager buildCatalogManager( SessionEnvironment environment) { CatalogStoreFactory catalogStoreFactory = - TableFactoryUtil.findAndCreateCatalogStoreFactory(configuration, userClassLoader); + ApiFactoryUtil.findAndCreateCatalogStoreFactory(configuration, userClassLoader); CatalogStoreFactory.Context catalogStoreFactoryContext = - TableFactoryUtil.buildCatalogStoreFactoryContext(configuration, userClassLoader); + ApiFactoryUtil.buildCatalogStoreFactoryContext(configuration, userClassLoader); catalogStoreFactory.open(catalogStoreFactoryContext); CatalogStoreHolder catalogStore = CatalogStoreHolder.newBuilder() diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java index 2bba1d76d696c..1ba1f34e6d856 100644 --- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java +++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java @@ -46,6 +46,7 @@ import org.apache.flink.table.delegation.Executor; import org.apache.flink.table.delegation.Planner; import org.apache.flink.table.expressions.Expression; +import org.apache.flink.table.factories.ApiFactoryUtil; import org.apache.flink.table.factories.CatalogStoreFactory; import org.apache.flink.table.factories.PlannerFactoryUtil; import org.apache.flink.table.factories.TableFactoryUtil; @@ -112,17 +113,21 @@ public static StreamTableEnvironment create( new ResourceManager(settings.getConfiguration(), userClassLoader); final ModuleManager moduleManager = new ModuleManager(); - final CatalogStoreFactory catalogStoreFactory = - TableFactoryUtil.findAndCreateCatalogStoreFactory( - settings.getConfiguration(), userClassLoader); - final CatalogStoreFactory.Context catalogStoreFactoryContext = - TableFactoryUtil.buildCatalogStoreFactoryContext( - settings.getConfiguration(), userClassLoader); - catalogStoreFactory.open(catalogStoreFactoryContext); - final CatalogStore catalogStore = - settings.getCatalogStore() != null - ? settings.getCatalogStore() - : catalogStoreFactory.createCatalogStore(); + final CatalogStore catalogStore; + final CatalogStoreFactory catalogStoreFactory; + if (settings.getCatalogStore().isPresent()) { + catalogStore = settings.getCatalogStore().get(); + catalogStoreFactory = null; + } else { + catalogStoreFactory = + ApiFactoryUtil.findAndCreateCatalogStoreFactory( + settings.getConfiguration(), userClassLoader); + final CatalogStoreFactory.Context catalogStoreFactoryContext = + ApiFactoryUtil.buildCatalogStoreFactoryContext( + settings.getConfiguration(), userClassLoader); + catalogStoreFactory.open(catalogStoreFactoryContext); + catalogStore = catalogStoreFactory.createCatalogStore(); + } final CatalogManager catalogManager = CatalogManager.newBuilder() diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/EnvironmentSettings.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/EnvironmentSettings.java index 67fcfcec46309..92a375bf0bf8f 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/EnvironmentSettings.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/EnvironmentSettings.java @@ -25,6 +25,7 @@ import org.apache.flink.table.catalog.CatalogStore; import org.apache.flink.table.expressions.SqlFactory; import org.apache.flink.table.functions.UserDefinedFunction; +import org.apache.flink.table.secret.SecretStore; import javax.annotation.Nullable; @@ -66,16 +67,19 @@ public class EnvironmentSettings { private final @Nullable CatalogStore catalogStore; private final @Nullable SqlFactory sqlFactory; + private final @Nullable SecretStore secretStore; private EnvironmentSettings( Configuration configuration, ClassLoader classLoader, CatalogStore catalogStore, - SqlFactory sqlFactory) { + SqlFactory sqlFactory, + SecretStore secretStore) { this.configuration = configuration; this.classLoader = classLoader; this.catalogStore = catalogStore; this.sqlFactory = sqlFactory; + this.secretStore = secretStore; } /** @@ -144,9 +148,8 @@ public ClassLoader getUserClassLoader() { } @Internal - @Nullable - public CatalogStore getCatalogStore() { - return catalogStore; + public Optional getCatalogStore() { + return Optional.ofNullable(catalogStore); } @Internal @@ -154,6 +157,11 @@ public Optional getSqlFactory() { return Optional.ofNullable(sqlFactory); } + @Internal + public Optional getSecretStore() { + return Optional.ofNullable(secretStore); + } + /** A builder for {@link EnvironmentSettings}. */ @PublicEvolving public static class Builder { @@ -163,6 +171,7 @@ public static class Builder { private @Nullable CatalogStore catalogStore; private @Nullable SqlFactory sqlFactory; + private @Nullable SecretStore secretStore; public Builder() {} @@ -251,12 +260,28 @@ public Builder withSqlFactory(SqlFactory sqlFactory) { return this; } + /** + * Specifies the {@link SecretStore} to be used for managing secrets in the {@link + * TableEnvironment}. + * + *

The secret store allows for secure storage and retrieval of sensitive configuration + * data such as credentials, tokens, and passwords. + * + * @param secretStore the secret store instance to use + * @return this builder + */ + public Builder withSecretStore(SecretStore secretStore) { + this.secretStore = secretStore; + return this; + } + /** Returns an immutable instance of {@link EnvironmentSettings}. */ public EnvironmentSettings build() { if (classLoader == null) { classLoader = Thread.currentThread().getContextClassLoader(); } - return new EnvironmentSettings(configuration, classLoader, catalogStore, sqlFactory); + return new EnvironmentSettings( + configuration, classLoader, catalogStore, sqlFactory, secretStore); } } } diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java index 6a40fa9c2384d..47c8b53dfd9fd 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java @@ -83,6 +83,7 @@ import org.apache.flink.table.expressions.ModelReferenceExpression; import org.apache.flink.table.expressions.TableReferenceExpression; import org.apache.flink.table.expressions.utils.ApiExpressionDefaultVisitor; +import org.apache.flink.table.factories.ApiFactoryUtil; import org.apache.flink.table.factories.CatalogStoreFactory; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.table.factories.PlannerFactoryUtil; @@ -119,6 +120,8 @@ import org.apache.flink.table.resource.ResourceManager; import org.apache.flink.table.resource.ResourceType; import org.apache.flink.table.resource.ResourceUri; +import org.apache.flink.table.secret.SecretStore; +import org.apache.flink.table.secret.SecretStoreFactory; import org.apache.flink.table.types.AbstractDataType; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.utils.DataTypeUtils; @@ -251,17 +254,38 @@ public static TableEnvironmentImpl create(EnvironmentSettings settings) { userClassLoader, ExecutorFactory.class, ExecutorFactory.DEFAULT_IDENTIFIER); final Executor executor = executorFactory.create(settings.getConfiguration()); - final CatalogStoreFactory catalogStoreFactory = - TableFactoryUtil.findAndCreateCatalogStoreFactory( - settings.getConfiguration(), userClassLoader); - final CatalogStoreFactory.Context context = - TableFactoryUtil.buildCatalogStoreFactoryContext( - settings.getConfiguration(), userClassLoader); - catalogStoreFactory.open(context); - final CatalogStore catalogStore = - settings.getCatalogStore() != null - ? settings.getCatalogStore() - : catalogStoreFactory.createCatalogStore(); + final CatalogStore catalogStore; + final CatalogStoreFactory catalogStoreFactory; + if (settings.getCatalogStore().isPresent()) { + catalogStore = settings.getCatalogStore().get(); + catalogStoreFactory = null; + } else { + catalogStoreFactory = + ApiFactoryUtil.findAndCreateCatalogStoreFactory( + settings.getConfiguration(), userClassLoader); + final CatalogStoreFactory.Context context = + ApiFactoryUtil.buildCatalogStoreFactoryContext( + settings.getConfiguration(), userClassLoader); + catalogStoreFactory.open(context); + catalogStore = catalogStoreFactory.createCatalogStore(); + } + + final SecretStore secretStore; + final SecretStoreFactory secretStoreFactory; + if (settings.getSecretStore().isPresent()) { + secretStore = settings.getSecretStore().get(); + secretStoreFactory = null; + } else { + secretStoreFactory = + ApiFactoryUtil.findAndCreateSecretStoreFactory( + settings.getConfiguration(), userClassLoader); + final SecretStoreFactory.Context secretStoreContext = + ApiFactoryUtil.buildSecretStoreFactoryContext( + settings.getConfiguration(), userClassLoader); + secretStoreFactory.open(secretStoreContext); + secretStore = secretStoreFactory.createSecretStore(); + } + // TODO (FLINK-38261): pass secret store to catalog manager for encryption/decryption // use configuration to init table config final TableConfig tableConfig = TableConfig.getDefault(); diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/factories/ApiFactoryUtil.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/factories/ApiFactoryUtil.java new file mode 100644 index 0000000000000..92486d94f1342 --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/factories/ApiFactoryUtil.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.factories; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.DelegatingConfiguration; +import org.apache.flink.table.catalog.CommonCatalogOptions; +import org.apache.flink.table.secret.CommonSecretOptions; +import org.apache.flink.table.secret.SecretStoreFactory; + +import java.util.Map; + +/** Utility for dealing with catalog store factories. */ +@Internal +public class ApiFactoryUtil { + + /** + * Finds and creates a {@link CatalogStoreFactory} using the provided {@link Configuration} and + * user classloader. + * + *

The configuration format should be as follows: + * + * 

{@code
+     * table.catalog-store.kind: {identifier}
+     * table.catalog-store.{identifier}.{param1}: xxx
+     * table.catalog-store.{identifier}.{param2}: xxx
+     * }
+ */ + public static CatalogStoreFactory findAndCreateCatalogStoreFactory( + Configuration configuration, ClassLoader classLoader) { + String identifier = configuration.get(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND); + + CatalogStoreFactory catalogStoreFactory = + FactoryUtil.discoverFactory(classLoader, CatalogStoreFactory.class, identifier); + + return catalogStoreFactory; + } + + /** + * Build a {@link CatalogStoreFactory.Context} for opening the {@link CatalogStoreFactory}. + * + *

The configuration format should be as follows: + * + * 

{@code
+     * table.catalog-store.kind: {identifier}
+     * table.catalog-store.{identifier}.{param1}: xxx
+     * table.catalog-store.{identifier}.{param2}: xxx
+     * }
+ */ + public static CatalogStoreFactory.Context buildCatalogStoreFactoryContext( + Configuration configuration, ClassLoader classLoader) { + String identifier = configuration.get(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND); + String catalogStoreOptionPrefix = + CommonCatalogOptions.TABLE_CATALOG_STORE_OPTION_PREFIX + identifier + "."; + Map options = + new DelegatingConfiguration(configuration, catalogStoreOptionPrefix).toMap(); + CatalogStoreFactory.Context context = + new FactoryUtil.DefaultCatalogStoreContext(options, configuration, classLoader); + + return context; + } + + public static SecretStoreFactory findAndCreateSecretStoreFactory( + Configuration configuration, ClassLoader classLoader) { + String identifier = configuration.get(CommonSecretOptions.TABLE_SECRET_STORE_KIND); + + SecretStoreFactory secretStoreFactory = + FactoryUtil.discoverFactory(classLoader, SecretStoreFactory.class, identifier); + + return secretStoreFactory; + } + + /** + * Build a {@link SecretStoreFactory.Context} for opening the {@link SecretStoreFactory}. + * + *

The configuration format should be as follows: + * + * 

{@code
+     * table.secret-store.kind: {identifier}
+     * table.secret-store.{identifier}.{param1}: xxx
+     * table.secret-store.{identifier}.{param2}: xxx
+     * }
+ */ + public static SecretStoreFactory.Context buildSecretStoreFactoryContext( + Configuration configuration, ClassLoader classLoader) { + String identifier = configuration.get(CommonSecretOptions.TABLE_SECRET_STORE_KIND); + String secretStoreOptionPrefix = + CommonSecretOptions.TABLE_SECRET_STORE_OPTION_PREFIX + identifier + "."; + Map options = + new DelegatingConfiguration(configuration, secretStoreOptionPrefix).toMap(); + SecretStoreFactory.Context context = + new FactoryUtil.DefaultSecretStoreContext(options, configuration, classLoader); + + return context; + } +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/factories/TableFactoryUtil.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/factories/TableFactoryUtil.java index ad681b5439441..ba10e4de707b7 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/factories/TableFactoryUtil.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/factories/TableFactoryUtil.java @@ -19,14 +19,11 @@ package org.apache.flink.table.factories; import org.apache.flink.annotation.Internal; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.DelegatingConfiguration; import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.config.TableConfigOptions; import org.apache.flink.table.catalog.Catalog; import org.apache.flink.table.catalog.CatalogTable; -import org.apache.flink.table.catalog.CommonCatalogOptions; import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.catalog.ResolvedCatalogTable; import org.apache.flink.table.catalog.listener.CatalogModificationListener; @@ -42,7 +39,6 @@ import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; /** Utility for dealing with {@link TableFactory} using the {@link TableFactoryService}. */ @@ -174,50 +170,4 @@ public ClassLoader getUserClassLoader() { })) .collect(Collectors.toList()); } - - /** - * Finds and creates a {@link CatalogStoreFactory} using the provided {@link Configuration} and - * user classloader. - * - *

The configuration format should be as follows: - * - * 

{@code
-     * table.catalog-store.kind: {identifier}
-     * table.catalog-store.{identifier}.{param1}: xxx
-     * table.catalog-store.{identifier}.{param2}: xxx
-     * }
- */ - public static CatalogStoreFactory findAndCreateCatalogStoreFactory( - Configuration configuration, ClassLoader classLoader) { - String identifier = configuration.get(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND); - - CatalogStoreFactory catalogStoreFactory = - FactoryUtil.discoverFactory(classLoader, CatalogStoreFactory.class, identifier); - - return catalogStoreFactory; - } - - /** - * Build a {@link CatalogStoreFactory.Context} for opening the {@link CatalogStoreFactory}. - * - *

The configuration format should be as follows: - * - * 

{@code
-     * table.catalog-store.kind: {identifier}
-     * table.catalog-store.{identifier}.{param1}: xxx
-     * table.catalog-store.{identifier}.{param2}: xxx
-     * }
- */ - public static CatalogStoreFactory.Context buildCatalogStoreFactoryContext( - Configuration configuration, ClassLoader classLoader) { - String identifier = configuration.get(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND); - String catalogStoreOptionPrefix = - CommonCatalogOptions.TABLE_CATALOG_STORE_OPTION_PREFIX + identifier + "."; - Map options = - new DelegatingConfiguration(configuration, catalogStoreOptionPrefix).toMap(); - CatalogStoreFactory.Context context = - new FactoryUtil.DefaultCatalogStoreContext(options, configuration, classLoader); - - return context; - } } diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/secret/GenericInMemorySecretStore.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/secret/GenericInMemorySecretStore.java new file mode 100644 index 0000000000000..7554bb789e621 --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/secret/GenericInMemorySecretStore.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.secret.exceptions.SecretNotFoundException; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A generic in-memory implementation of both {@link ReadableSecretStore} and {@link + * WritableSecretStore}. + * + *

This implementation stores secrets in memory as immutable Map objects. It is suitable for + * testing and development purposes but should not be used in production environments as secrets are + * not encrypted. + */ +@Internal +public class GenericInMemorySecretStore implements ReadableSecretStore, WritableSecretStore { + + private final Map> secrets; + + public GenericInMemorySecretStore() { + this.secrets = new HashMap<>(); + } + + @Override + public Map getSecret(String secretId) throws SecretNotFoundException { + checkNotNull(secretId, "Secret ID cannot be null"); + + Map secretData = secrets.get(secretId); + if (secretData == null) { + throw new SecretNotFoundException( + String.format("Secret with ID '%s' not found", secretId)); + } + + return secretData; + } + + @Override + public String storeSecret(Map secretData) { + checkNotNull(secretData, "Secret data cannot be null"); + + String secretId = UUID.randomUUID().toString(); + secrets.put(secretId, Collections.unmodifiableMap(new HashMap<>(secretData))); + return secretId; + } + + @Override + public void removeSecret(String secretId) { + checkNotNull(secretId, "Secret ID cannot be null"); + secrets.remove(secretId); + } + + @Override + public void updateSecret(String secretId, Map newSecretData) + throws SecretNotFoundException { + checkNotNull(secretId, "Secret ID cannot be null"); + checkNotNull(newSecretData, "New secret data cannot be null"); + + if (!secrets.containsKey(secretId)) { + throw new SecretNotFoundException( + String.format("Secret with ID '%s' not found", secretId)); + } + + secrets.put(secretId, Collections.unmodifiableMap(new HashMap<>(newSecretData))); + } + + /** Clears all secrets from the store (for testing purposes). */ + void clear() { + secrets.clear(); + } +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/secret/GenericInMemorySecretStoreFactory.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/secret/GenericInMemorySecretStoreFactory.java new file mode 100644 index 0000000000000..bbccfd4ec2f4c --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/secret/GenericInMemorySecretStoreFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.table.catalog.exceptions.CatalogException; + +import java.util.Set; + +/** + * Factory for creating {@link GenericInMemorySecretStore} instances. + * + *

This factory creates in-memory secret stores that are suitable for testing and development + * purposes. Secrets are stored in plaintext JSON format in memory and are not persisted. + */ +@Internal +public class GenericInMemorySecretStoreFactory implements SecretStoreFactory { + + public static final String IDENTIFIER = "generic_in_memory"; + + @Override + public String factoryIdentifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + return Set.of(); + } + + @Override + public Set> optionalOptions() { + return Set.of(); + } + + @Override + public SecretStore createSecretStore() { + return new GenericInMemorySecretStore(); + } + + @Override + public void open(Context context) {} + + @Override + public void close() throws CatalogException {} +} diff --git a/flink-table/flink-table-api-java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-table/flink-table-api-java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory index d86ead55ad642..8191a71a678a2 100644 --- a/flink-table/flink-table-api-java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory +++ b/flink-table/flink-table-api-java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -16,3 +16,4 @@ org.apache.flink.table.catalog.GenericInMemoryCatalogFactory org.apache.flink.table.catalog.GenericInMemoryCatalogStoreFactory org.apache.flink.table.catalog.FileCatalogStoreFactory +org.apache.flink.table.secret.GenericInMemorySecretStoreFactory diff --git a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/factories/ApiFactoryUtilTest.java b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/factories/ApiFactoryUtilTest.java new file mode 100644 index 0000000000000..e1ea22a6f5771 --- /dev/null +++ b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/factories/ApiFactoryUtilTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.factories; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.catalog.CommonCatalogOptions; +import org.apache.flink.table.catalog.FileCatalogStoreFactory; +import org.apache.flink.table.catalog.GenericInMemoryCatalogStoreFactory; +import org.apache.flink.table.secret.CommonSecretOptions; +import org.apache.flink.table.secret.GenericInMemorySecretStoreFactory; +import org.apache.flink.table.secret.SecretStoreFactory; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.File; +import java.util.Map; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link ApiFactoryUtil}. */ +class ApiFactoryUtilTest { + + @ParameterizedTest(name = "kind={0}, expectedFactory={1}") + @MethodSource("catalogStoreFactoryTestParameters") + void testFindAndCreateCatalogStoreFactory(String kind, Class expectedFactoryClass) { + Configuration configuration = new Configuration(); + if (kind != null) { + configuration.set(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND, kind); + } + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + CatalogStoreFactory factory = + ApiFactoryUtil.findAndCreateCatalogStoreFactory(configuration, classLoader); + + assertThat(factory).isInstanceOf(expectedFactoryClass); + } + + @Test + void testBuildCatalogStoreFactoryContext(@TempDir File tempFolder) { + Configuration configuration = new Configuration(); + configuration.set(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND, "file"); + configuration.setString("table.catalog-store.file.path", tempFolder.getAbsolutePath()); + configuration.setString("table.catalog-store.file.option1", "value1"); + configuration.setString("table.catalog-store.file.option2", "value2"); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + CatalogStoreFactory.Context context = + ApiFactoryUtil.buildCatalogStoreFactoryContext(configuration, classLoader); + + assertThat(context).isNotNull(); + assertThat(context.getOptions()) + .containsExactlyInAnyOrderEntriesOf( + Map.of( + "path", tempFolder.getAbsolutePath(), + "option1", "value1", + "option2", "value2")); + assertThat(context.getConfiguration()).isEqualTo(configuration); + assertThat(context.getClassLoader()).isEqualTo(classLoader); + } + + @Test + void testBuildCatalogStoreFactoryContextWithGenericInMemory() { + Configuration configuration = new Configuration(); + configuration.set(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND, "generic_in_memory"); + configuration.setString("table.catalog-store.generic_in_memory.option1", "value1"); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + CatalogStoreFactory.Context context = + ApiFactoryUtil.buildCatalogStoreFactoryContext(configuration, classLoader); + + assertThat(context).isNotNull(); + assertThat(context.getOptions()) + .containsExactlyInAnyOrderEntriesOf(Map.of("option1", "value1")); + assertThat(context.getConfiguration()).isEqualTo(configuration); + assertThat(context.getClassLoader()).isEqualTo(classLoader); + } + + @Test + void testBuildCatalogStoreFactoryContextWithoutOptions() { + Configuration configuration = new Configuration(); + configuration.set(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND, "generic_in_memory"); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + CatalogStoreFactory.Context context = + ApiFactoryUtil.buildCatalogStoreFactoryContext(configuration, classLoader); + + assertThat(context).isNotNull(); + assertThat(context.getOptions()).isEmpty(); + assertThat(context.getConfiguration()).isEqualTo(configuration); + assertThat(context.getClassLoader()).isEqualTo(classLoader); + } + + @Test + void testBuildCatalogStoreFactoryContextOnlyExtractsRelevantOptions() { + Configuration configuration = new Configuration(); + configuration.set(CommonCatalogOptions.TABLE_CATALOG_STORE_KIND, "file"); + configuration.setString("table.catalog-store.file.path", "/test/path"); + configuration.setString("table.catalog-store.file.option1", "value1"); + configuration.setString("table.catalog-store.other.irrelevant", "should-not-appear"); + configuration.setString("other.config.key", "should-not-appear"); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + CatalogStoreFactory.Context context = + ApiFactoryUtil.buildCatalogStoreFactoryContext(configuration, classLoader); + + assertThat(context).isNotNull(); + assertThat(context.getOptions()) + .containsExactlyInAnyOrderEntriesOf( + Map.of("path", "/test/path", "option1", "value1")); + } + + @ParameterizedTest(name = "kind={0}, expectedFactory={1}") + @MethodSource("secretStoreFactoryTestParameters") + void testFindAndCreateSecretStoreFactory(String kind, Class expectedFactoryClass) { + Configuration configuration = new Configuration(); + if (kind != null) { + configuration.set(CommonSecretOptions.TABLE_SECRET_STORE_KIND, kind); + } + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + SecretStoreFactory factory = + ApiFactoryUtil.findAndCreateSecretStoreFactory(configuration, classLoader); + + assertThat(factory).isInstanceOf(expectedFactoryClass); + } + + @Test + void testBuildSecretStoreFactoryContext() { + Configuration configuration = new Configuration(); + configuration.set(CommonSecretOptions.TABLE_SECRET_STORE_KIND, "generic_in_memory"); + configuration.setString("table.secret-store.generic_in_memory.option1", "value1"); + configuration.setString("table.secret-store.generic_in_memory.option2", "value2"); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + SecretStoreFactory.Context context = + ApiFactoryUtil.buildSecretStoreFactoryContext(configuration, classLoader); + + assertThat(context).isNotNull(); + assertThat(context.getOptions()) + .containsExactlyInAnyOrderEntriesOf( + Map.of("option1", "value1", "option2", "value2")); + assertThat(context.getConfiguration()).isEqualTo(configuration); + assertThat(context.getClassLoader()).isEqualTo(classLoader); + } + + @Test + void testBuildSecretStoreFactoryContextWithoutOptions() { + Configuration configuration = new Configuration(); + configuration.set(CommonSecretOptions.TABLE_SECRET_STORE_KIND, "generic_in_memory"); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + SecretStoreFactory.Context context = + ApiFactoryUtil.buildSecretStoreFactoryContext(configuration, classLoader); + + assertThat(context).isNotNull(); + assertThat(context.getOptions()).isEmpty(); + assertThat(context.getConfiguration()).isEqualTo(configuration); + assertThat(context.getClassLoader()).isEqualTo(classLoader); + } + + @Test + void testBuildSecretStoreFactoryContextOnlyExtractsRelevantOptions() { + Configuration configuration = new Configuration(); + configuration.set(CommonSecretOptions.TABLE_SECRET_STORE_KIND, "generic_in_memory"); + configuration.setString("table.secret-store.generic_in_memory.option1", "value1"); + configuration.setString("table.secret-store.generic_in_memory.option2", "value2"); + configuration.setString("table.secret-store.other.irrelevant", "should-not-appear"); + configuration.setString("other.config.key", "should-not-appear"); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + SecretStoreFactory.Context context = + ApiFactoryUtil.buildSecretStoreFactoryContext(configuration, classLoader); + + assertThat(context).isNotNull(); + assertThat(context.getOptions()) + .containsExactlyInAnyOrderEntriesOf( + Map.of("option1", "value1", "option2", "value2")); + } + + private static Stream catalogStoreFactoryTestParameters() { + return Stream.of( + Arguments.of("generic_in_memory", GenericInMemoryCatalogStoreFactory.class), + Arguments.of("file", FileCatalogStoreFactory.class), + Arguments.of(null, GenericInMemoryCatalogStoreFactory.class)); + } + + private static Stream secretStoreFactoryTestParameters() { + return Stream.of( + Arguments.of("generic_in_memory", GenericInMemorySecretStoreFactory.class), + Arguments.of(null, GenericInMemorySecretStoreFactory.class)); + } +} diff --git a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/secret/GenericInMemorySecretStoreFactoryTest.java b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/secret/GenericInMemorySecretStoreFactoryTest.java new file mode 100644 index 0000000000000..a935114330659 --- /dev/null +++ b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/secret/GenericInMemorySecretStoreFactoryTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.factories.FactoryUtil.DefaultSecretStoreContext; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Test for {@link GenericInMemorySecretStoreFactory}. */ +class GenericInMemorySecretStoreFactoryTest { + + @Test + void testSecretStoreInit() { + String factoryIdentifier = GenericInMemorySecretStoreFactory.IDENTIFIER; + Map options = new HashMap<>(); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + final DefaultSecretStoreContext discoveryContext = + new DefaultSecretStoreContext(options, null, classLoader); + final SecretStoreFactory factory = + FactoryUtil.discoverFactory( + classLoader, SecretStoreFactory.class, factoryIdentifier); + try { + factory.open(discoveryContext); + SecretStore secretStore = factory.createSecretStore(); + assertThat(secretStore instanceof GenericInMemorySecretStore).isTrue(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + factory.close(); + } + } + + @Test + void testFactoryIdentifier() { + GenericInMemorySecretStoreFactory factory = new GenericInMemorySecretStoreFactory(); + assertThat(factory.factoryIdentifier()).isEqualTo("generic_in_memory"); + } + + @Test + void testRequiredAndOptionalOptions() { + GenericInMemorySecretStoreFactory factory = new GenericInMemorySecretStoreFactory(); + assertThat(factory.requiredOptions()).isEmpty(); + assertThat(factory.optionalOptions()).isEmpty(); + } +} diff --git a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/secret/GenericInMemorySecretStoreTest.java b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/secret/GenericInMemorySecretStoreTest.java new file mode 100644 index 0000000000000..53674e1e5c03f --- /dev/null +++ b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/secret/GenericInMemorySecretStoreTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.table.secret.exceptions.SecretNotFoundException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +/** Test for {@link GenericInMemorySecretStore}. */ +class GenericInMemorySecretStoreTest { + + private GenericInMemorySecretStore secretStore; + + @BeforeEach + void setUp() { + secretStore = new GenericInMemorySecretStore(); + } + + @Test + void testStoreAndGetSecret() throws SecretNotFoundException { + Map secretData = Map.of("username", "testuser", "password", "testpass"); + + String secretId = secretStore.storeSecret(secretData); + assertThat(secretId).isNotNull(); + + Map retrievedSecret = secretStore.getSecret(secretId); + assertThat(retrievedSecret).isNotNull(); + assertThat(retrievedSecret.get("username")).isEqualTo("testuser"); + assertThat(retrievedSecret.get("password")).isEqualTo("testpass"); + } + + @Test + void testGetNonExistentSecret() { + assertThatThrownBy(() -> secretStore.getSecret("non-existent-id")) + .isInstanceOf(SecretNotFoundException.class) + .hasMessageContaining("Secret with ID 'non-existent-id' not found"); + } + + @Test + void testGetSecretWithNullId() { + assertThatThrownBy(() -> secretStore.getSecret(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("Secret ID cannot be null"); + } + + @Test + void testStoreSecretWithNullData() { + assertThatThrownBy(() -> secretStore.storeSecret(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("Secret data cannot be null"); + } + + @Test + void testRemoveSecret() throws SecretNotFoundException { + Map secretData = Map.of("key", "value"); + + String secretId = secretStore.storeSecret(secretData); + assertThat(secretStore.getSecret(secretId)).isNotNull(); + + secretStore.removeSecret(secretId); + assertThatThrownBy(() -> secretStore.getSecret(secretId)) + .isInstanceOf(SecretNotFoundException.class) + .hasMessageContaining("Secret with ID '" + secretId + "' not found"); + } + + @Test + void testRemoveSecretWithNullId() { + assertThatThrownBy(() -> secretStore.removeSecret(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("Secret ID cannot be null"); + } + + @Test + void testRemoveNonExistentSecret() { + // Should not throw exception, just silently remove nothing + assertDoesNotThrow(() -> secretStore.removeSecret("non-existent-id")); + } + + @Test + void testUpdateSecret() throws SecretNotFoundException { + Map originalData = Map.of("username", "olduser", "password", "oldpass"); + + String secretId = secretStore.storeSecret(originalData); + + Map updatedData = Map.of("username", "newuser", "password", "newpass"); + + secretStore.updateSecret(secretId, updatedData); + + Map retrievedSecret = secretStore.getSecret(secretId); + assertThat(retrievedSecret.get("username")).isEqualTo("newuser"); + assertThat(retrievedSecret.get("password")).isEqualTo("newpass"); + } + + @Test + void testUpdateNonExistentSecret() { + Map secretData = Map.of("key", "value"); + + assertThatThrownBy(() -> secretStore.updateSecret("non-existent-id", secretData)) + .isInstanceOf(SecretNotFoundException.class) + .hasMessageContaining("Secret with ID 'non-existent-id' not found"); + } + + @Test + void testUpdateSecretWithNullId() { + Map secretData = Map.of("key", "value"); + + assertThatThrownBy(() -> secretStore.updateSecret(null, secretData)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("Secret ID cannot be null"); + } + + @Test + void testUpdateSecretWithNullData() { + Map originalData = Map.of("key", "value"); + String secretId = secretStore.storeSecret(originalData); + + assertThatThrownBy(() -> secretStore.updateSecret(secretId, null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("New secret data cannot be null"); + } + + @Test + void testClear() { + Map secretData1 = Map.of("key1", "value1"); + String secretId1 = secretStore.storeSecret(secretData1); + + Map secretData2 = Map.of("key2", "value2"); + String secretId2 = secretStore.storeSecret(secretData2); + + secretStore.clear(); + + assertThatThrownBy(() -> secretStore.getSecret(secretId1)) + .isInstanceOf(SecretNotFoundException.class); + assertThatThrownBy(() -> secretStore.getSecret(secretId2)) + .isInstanceOf(SecretNotFoundException.class); + } + + @Test + void testStoreEmptySecret() throws SecretNotFoundException { + Map emptyData = Map.of(); + String secretId = secretStore.storeSecret(emptyData); + + Map retrievedSecret = secretStore.getSecret(secretId); + assertThat(retrievedSecret).isNotNull(); + assertThat(retrievedSecret).isEmpty(); + } + + @Test + void testStoreMultipleSecrets() throws SecretNotFoundException { + Map secret1 = Map.of("user1", "pass1"); + + Map secret2 = Map.of("user2", "pass2"); + + String secretId1 = secretStore.storeSecret(secret1); + String secretId2 = secretStore.storeSecret(secret2); + + assertThat(secretId1).isNotEqualTo(secretId2); + + Map retrieved1 = secretStore.getSecret(secretId1); + Map retrieved2 = secretStore.getSecret(secretId2); + + assertThat(retrieved1.get("user1")).isEqualTo("pass1"); + assertThat(retrieved2.get("user2")).isEqualTo("pass2"); + } +} diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImpl.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImpl.scala index 7e86c5bf256cd..c8cc4e99463d2 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImpl.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImpl.scala @@ -28,7 +28,7 @@ import org.apache.flink.table.catalog._ import org.apache.flink.table.connector.ChangelogMode import org.apache.flink.table.delegation.{Executor, Planner} import org.apache.flink.table.expressions.Expression -import org.apache.flink.table.factories.{PlannerFactoryUtil, TableFactoryUtil} +import org.apache.flink.table.factories.{ApiFactoryUtil, PlannerFactoryUtil, TableFactoryUtil} import org.apache.flink.table.functions.{AggregateFunction, TableAggregateFunction, TableFunction, UserDefinedFunctionHelper} import org.apache.flink.table.legacy.sources.TableSource import org.apache.flink.table.module.ModuleManager @@ -250,14 +250,19 @@ object StreamTableEnvironmentImpl { val resourceManager = new ResourceManager(settings.getConfiguration, userClassLoader) val moduleManager = new ModuleManager - val catalogStoreFactory = - TableFactoryUtil.findAndCreateCatalogStoreFactory(settings.getConfiguration, userClassLoader) - val catalogStoreFactoryContext = - TableFactoryUtil.buildCatalogStoreFactoryContext(settings.getConfiguration, userClassLoader) - catalogStoreFactory.open(catalogStoreFactoryContext) - val catalogStore = - if (settings.getCatalogStore != null) settings.getCatalogStore - else catalogStoreFactory.createCatalogStore() + val (catalogStore, catalogStoreFactory) = + if (settings.getCatalogStore.isPresent) { + (settings.getCatalogStore.get(), null) + } else { + val factory = + ApiFactoryUtil.findAndCreateCatalogStoreFactory( + settings.getConfiguration, + userClassLoader) + val factoryContext = + ApiFactoryUtil.buildCatalogStoreFactoryContext(settings.getConfiguration, userClassLoader) + factory.open(factoryContext) + (factory.createCatalogStore(), factory) + } val catalogManager = CatalogManager.newBuilder .classLoader(userClassLoader) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/FactoryUtil.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/FactoryUtil.java index 691397c08d4bd..594a7bf1d5d9b 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/FactoryUtil.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/FactoryUtil.java @@ -42,6 +42,7 @@ import org.apache.flink.table.legacy.factories.TableFactory; import org.apache.flink.table.ml.ModelProvider; import org.apache.flink.table.module.Module; +import org.apache.flink.table.secret.SecretStoreFactory; import org.apache.flink.table.utils.EncodingUtils; import org.apache.flink.table.watermark.WatermarkEmitStrategy; import org.apache.flink.util.Preconditions; @@ -1411,6 +1412,41 @@ public ClassLoader getClassLoader() { } } + /** Default implementation of {@link SecretStoreFactory.Context}. */ + @Internal + public static class DefaultSecretStoreContext implements SecretStoreFactory.Context { + + private Map options; + + private ReadableConfig configuration; + + private ClassLoader classLoader; + + public DefaultSecretStoreContext( + Map options, + ReadableConfig configuration, + ClassLoader classLoader) { + this.options = options; + this.configuration = configuration; + this.classLoader = classLoader; + } + + @Override + public Map getOptions() { + return options; + } + + @Override + public ReadableConfig getConfiguration() { + return configuration; + } + + @Override + public ClassLoader getClassLoader() { + return classLoader; + } + } + /** Default implementation of {@link ModuleFactory.Context}. */ @Internal public static class DefaultModuleContext implements ModuleFactory.Context { diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/CommonSecretOptions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/CommonSecretOptions.java new file mode 100644 index 0000000000000..461909a6cd187 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/CommonSecretOptions.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +/** A collection of {@link ConfigOption} which are used for secret store configuration. */ +@Internal +public class CommonSecretOptions { + + public static final String DEFAULT_SECRET_STORE_KIND = "generic_in_memory"; + public static final ConfigOption TABLE_SECRET_STORE_KIND = + ConfigOptions.key("table.secret-store.kind") + .stringType() + .defaultValue(DEFAULT_SECRET_STORE_KIND) + .withDescription( + "The kind of secret store to be used. Out of the box, 'generic_in_memory' option is supported. " + + "Implementations can provide custom secret stores for different backends " + + "(e.g., cloud-specific secret managers)."); + + /** Used to filter the specific options for secret store. */ + public static final String TABLE_SECRET_STORE_OPTION_PREFIX = "table.secret-store."; +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/ReadableSecretStore.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/ReadableSecretStore.java new file mode 100644 index 0000000000000..70dd48e21f979 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/ReadableSecretStore.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.secret.exceptions.SecretNotFoundException; + +import java.util.Map; + +/** + * Interface for retrieving secrets from a secret store. + * + *

This interface enables read-only access to stored secrets, allowing applications to retrieve + * sensitive configuration data such as credentials, API tokens, and passwords. + * + *

Implementations of this interface should ensure secure retrieval and handling of secret data. + */ +@PublicEvolving +public interface ReadableSecretStore extends SecretStore { + + /** + * Retrieves a secret from the store by its identifier. + * + * @param secretId the unique identifier of the secret to retrieve + * @return a map containing the secret data as key-value pairs + * @throws SecretNotFoundException if the secret with the given identifier does not exist + */ + Map getSecret(String secretId) throws SecretNotFoundException; +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/SecretStore.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/SecretStore.java new file mode 100644 index 0000000000000..89af1d469e1b8 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/SecretStore.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * Base marker interface for secret store implementations. + * + *

This interface serves as the common base for both {@link ReadableSecretStore} and {@link + * WritableSecretStore}, allowing for flexible secret management implementations. + * + *

Secret stores are used to manage sensitive configuration data (credentials, tokens, passwords, + * etc.) in Flink SQL and Table API applications. + */ +@PublicEvolving +public interface SecretStore {} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/SecretStoreFactory.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/SecretStoreFactory.java new file mode 100644 index 0000000000000..1fd40b8e78be0 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/SecretStoreFactory.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.factories.Factory; +import org.apache.flink.table.secret.exceptions.SecretException; + +import java.util.Map; + +/** Factory for creating SecretStore instances. */ +@PublicEvolving +public interface SecretStoreFactory extends Factory { + + /** Creates a SecretStore instance. */ + SecretStore createSecretStore(); + + /** Initialize secret store. */ + void open(Context context) throws SecretException; + + /** Close secret store. */ + void close() throws CatalogException; + + /** Context for creating a SecretStore. */ + @PublicEvolving + interface Context { + /** + * Returns the options with which the secret store is created. + * + *

An implementation should perform validation of these options. + */ + Map getOptions(); + + /** Gives read-only access to the configuration of the current session. */ + ReadableConfig getConfiguration(); + + /** + * Returns the class loader of the current session. + * + *

The class loader is in particular useful for discovering further (nested) factories. + */ + ClassLoader getClassLoader(); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/WritableSecretStore.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/WritableSecretStore.java new file mode 100644 index 0000000000000..db5037b7b2f53 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/WritableSecretStore.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.secret.exceptions.SecretNotFoundException; + +import java.util.Map; + +/** + * Interface for storing, updating, and removing secrets in a secret store. + * + *

This interface provides write operations for managing secrets, including adding new secrets, + * updating existing ones, and removing secrets that are no longer needed. + * + *

Implementations should ensure that secret operations are performed securely and, where + * applicable, atomically. + */ +@PublicEvolving +public interface WritableSecretStore extends SecretStore { + + /** + * Stores a new secret in the secret store. + * + * @param secretData a map containing the secret data as key-value pairs to be stored + * @return a unique identifier for the stored secret + */ + String storeSecret(Map secretData); + + /** + * Removes a secret from the secret store. + * + * @param secretId the unique identifier of the secret to remove + */ + void removeSecret(String secretId); + + /** + * Atomically updates an existing secret with new data. + * + *

This operation replaces the entire secret data with the provided new data. + * + * @param secretId the unique identifier of the secret to update + * @param newSecretData a map containing the new secret data as key-value pairs + * @throws SecretNotFoundException if the secret with the given identifier does not exist + */ + void updateSecret(String secretId, Map newSecretData) + throws SecretNotFoundException; +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/exceptions/SecretException.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/exceptions/SecretException.java new file mode 100644 index 0000000000000..b856938ba6a54 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/exceptions/SecretException.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret.exceptions; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * Base exception for all secret-related errors. + * + *

This exception serves as the parent class for all secret related exceptions, providing a + * common type for handling errors that occur during secret management operations. + */ +@PublicEvolving +public class SecretException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + /** + * Constructs a new SecretException with the specified detail message. + * + * @param message the detail message explaining the reason for the exception + */ + public SecretException(String message) { + super(message); + } + + /** + * Constructs a new SecretException with the specified detail message and cause. + * + * @param message the detail message explaining the reason for the exception + * @param cause the cause of the exception + */ + public SecretException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new SecretException with the specified cause. + * + * @param cause the cause of the exception + */ + public SecretException(Throwable cause) { + super(cause); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/exceptions/SecretNotFoundException.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/exceptions/SecretNotFoundException.java new file mode 100644 index 0000000000000..a773ea7e1e685 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/secret/exceptions/SecretNotFoundException.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.secret.exceptions; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * Exception thrown when a requested secret cannot be found in the secret store. + * + *

This exception is typically thrown by {@link ReadableSecretStore#getSecret(String)} or {@link + * WritableSecretStore#updateSecret(String, java.util.Map)} when attempting to access or modify a + * secret that does not exist. + */ +@PublicEvolving +public class SecretNotFoundException extends Exception { + + private static final long serialVersionUID = 1L; + + /** + * Constructs a new SecretNotFoundException with the specified detail message. + * + * @param message the detail message explaining the reason for the exception + */ + public SecretNotFoundException(String message) { + super(message); + } + + /** + * Constructs a new SecretNotFoundException with the specified detail message and cause. + * + * @param message the detail message explaining the reason for the exception + * @param cause the cause of the exception + */ + public SecretNotFoundException(String message, Throwable cause) { + super(message, cause); + } +}