From f34d493fbaf70c5e14f2c2768f30f566669dec4f Mon Sep 17 00:00:00 2001 From: Georg Rollinger Date: Wed, 12 May 2021 18:55:57 +0200 Subject: [PATCH] Use ConfigMapList's resourceVersion for watching (PropertySource) Rationale: avoid ERROR notifications during rolling upgrades. Using the greatest `resourceVersion` of all the `ConfigMap`s returned within the `ConfigMapList` works as expected for *fresh* deployments. But, when performing a *rolling upgrade* (and depending on the upgrade strategy), the watcher happens to frequently stop after having received an `ERROR` notification: >>> [ERROR] [KubernetesConfigMapWatcher] [] Kubernetes API returned an error for a ConfigMap watch event: ConfigMapWatchEvent{type=ERROR, object=ConfigMap{metadata=Metadata{name='null', namespace='null', uid='null', labels={}, resourceVersion=null}, data={}}} >>> What's actually streamed in that case is a `Status` object such as: ```json { "type": "ERROR", "object": { "kind": "Status", "apiVersion": "v1", "metadata": {}, "status": "Expired", "message": "too old resource version: 123 (456)", "reason": "Gone", "code": 410 } } ``` A few references: * https://github.com/abonas/kubeclient/issues/452 * https://www.baeldung.com/java-kubernetes-watch#1-resource-versions It's possible to recover by adding some logic to reinstall the watcher starting with the newly advertised `resourceVersion`, but this may be avoided at all by starting the initial watch at the `resourceVersion` of the `ConfigMapList` itself: this one won't expire. The proposed implementation consists in storing last received `resourceVersion` as an additional `PropertySource` (through a dedicated name and version key) and later using it when installing the watcher. Co-authored-by: Regis Desgroppes --- .../client/v1/configmaps/ConfigMapList.java | 3 ++- .../configuration/KubernetesConfigMapWatcher.java | 6 +++--- .../KubernetesConfigurationClient.java | 13 ++++++++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/client/v1/configmaps/ConfigMapList.java b/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/client/v1/configmaps/ConfigMapList.java index 9f4785e83..aef56a497 100644 --- a/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/client/v1/configmaps/ConfigMapList.java +++ b/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/client/v1/configmaps/ConfigMapList.java @@ -16,6 +16,7 @@ package io.micronaut.kubernetes.client.v1.configmaps; import io.micronaut.core.annotation.Introspected; +import io.micronaut.kubernetes.client.v1.KubernetesObject; import java.util.Collections; import java.util.List; @@ -28,7 +29,7 @@ * @since 1.0.0 */ @Introspected -public class ConfigMapList { +public class ConfigMapList extends KubernetesObject { private List items; diff --git a/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigMapWatcher.java b/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigMapWatcher.java index a057fb693..52fd645ef 100644 --- a/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigMapWatcher.java +++ b/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigMapWatcher.java @@ -37,7 +37,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; -import static io.micronaut.kubernetes.configuration.KubernetesConfigurationClient.KUBERNETES_CONFIG_MAP_NAME_SUFFIX; +import static io.micronaut.kubernetes.configuration.KubernetesConfigurationClient.KUBERNETES_CONFIG_MAP_LIST_NAME; import static io.micronaut.kubernetes.util.KubernetesUtils.computePodLabelSelector; /** @@ -104,8 +104,8 @@ private long computeLastResourceVersion() { long lastResourceVersion = this.environment .getPropertySources() .stream() - .filter(propertySource -> propertySource.getName().endsWith(KUBERNETES_CONFIG_MAP_NAME_SUFFIX)) - .map(propertySource -> propertySource.get(KubernetesConfigurationClient.CONFIG_MAP_RESOURCE_VERSION)) + .filter(propertySource -> propertySource.getName().equals(KUBERNETES_CONFIG_MAP_LIST_NAME)) + .map(propertySource -> propertySource.get(KubernetesConfigurationClient.CONFIG_MAP_LIST_RESOURCE_VERSION)) .map(o -> Long.parseLong(o.toString())) .max(Long::compareTo) .orElse(0L); diff --git a/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigurationClient.java b/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigurationClient.java index 46ac30910..7badb75ed 100644 --- a/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigurationClient.java +++ b/kubernetes-discovery-client/src/main/java/io/micronaut/kubernetes/configuration/KubernetesConfigurationClient.java @@ -59,7 +59,9 @@ @BootstrapContextCompatible public class KubernetesConfigurationClient implements ConfigurationClient { + public static final String CONFIG_MAP_LIST_RESOURCE_VERSION = "configMapListResourceVersion"; public static final String CONFIG_MAP_RESOURCE_VERSION = "configMapResourceVersion"; + public static final String KUBERNETES_CONFIG_MAP_LIST_NAME = "Kubernetes ConfigMapList"; public static final String KUBERNETES_CONFIG_MAP_NAME_SUFFIX = " (Kubernetes ConfigMap)"; public static final String KUBERNETES_SECRET_NAME_SUFFIX = " (Kubernetes Secret)"; @@ -145,7 +147,7 @@ private Flowable getPropertySourcesFromConfigMaps() { Predicate excludesFilter = KubernetesUtils.getExcludesFilter(configuration.getConfigMaps().getExcludes()); Map labels = configuration.getConfigMaps().getLabels(); - return computePodLabelSelector(client, configuration.getConfigMaps().getPodLabels(), configuration.getNamespace(), labels) + final Flowable configMapListFlowable = computePodLabelSelector(client, configuration.getConfigMaps().getPodLabels(), configuration.getNamespace(), labels) .flatMap(labelSelector -> client.listConfigMaps(configuration.getNamespace(), labelSelector)) .doOnError(throwable -> LOG.error("Error while trying to list all Kubernetes ConfigMaps in the namespace [" + configuration.getNamespace() + "]", throwable)) .onErrorReturn(throwable -> new ConfigMapList()) @@ -153,7 +155,11 @@ private Flowable getPropertySourcesFromConfigMaps() { if (LOG.isDebugEnabled()) { LOG.debug("Found {} config maps. Applying includes/excludes filters (if any)", configMapList.getItems().size()); } - }) + }).cache(); + + final Flowable listResourceVersionPropertySource = configMapListFlowable.map(list -> PropertySource.of(KUBERNETES_CONFIG_MAP_LIST_NAME, Collections.singletonMap(CONFIG_MAP_LIST_RESOURCE_VERSION, list.getMetadata().getResourceVersion()), EnvironmentPropertySource.POSITION + 100)); + + return configMapListFlowable .flatMapIterable(ConfigMapList::getItems) .filter(includesFilter) .filter(excludesFilter) @@ -162,7 +168,8 @@ private Flowable getPropertySourcesFromConfigMaps() { LOG.debug("Adding config map with name {}", configMap.getMetadata().getName()); } }) - .map(KubernetesUtils::configMapAsPropertySource); + .map(KubernetesUtils::configMapAsPropertySource) + .mergeWith(listResourceVersionPropertySource); } private Flowable getPropertySourcesFromSecrets() {