From ce8a360bf832f74200d576ef44dc74cd5686a0f4 Mon Sep 17 00:00:00 2001 From: Philip Dakowitz Date: Mon, 11 Jul 2022 16:44:06 +0200 Subject: [PATCH] feat(metrics): add metrics (#79) * include and expose default metrics * update dependencies * add metrics for scheduled lifetime tasks and for the docker registry client * add more metrics * add metrics to deploymentmanager * add metrics to deploymentmanager * add more metrics to the k8s parts * add websocket metrics * fix metric names * release name is human-readable Co-authored-by: Tom Schoener --- README.md | 2 +- pom.xml | 35 +-- .../ScheduledLifetimeController.java | 75 +++++-- .../oneko/docker/DockerRegistryPolling.java | 20 +- .../v2/DockerRegistryClientFactory.java | 58 ++++- .../docker/v2/DockerRegistryV2Client.java | 24 +- .../v2/DockerRegistryVersionChecker.java | 24 +- .../metrics/DockerRegistryClientMetrics.java | 28 +++ .../docker/v2/metrics/MetersPerRegistry.java | 58 +++++ .../java/io/oneko/event/EventDispatcher.java | 13 +- src/main/java/io/oneko/helm/HelmCharts.java | 56 +++-- src/main/java/io/oneko/helm/HelmCommands.java | 207 ++++++++++++++++++ .../oneko/helm/HelmRegistryInitializer.java | 8 +- .../helm/rest/HelmRegistryController.java | 15 +- .../io/oneko/helm/util/HelmCommandUtils.java | 153 ------------- .../impl/DeploymentManagerImpl.java | 65 +++++- .../impl/DeploymentStatusWatcher.java | 63 ++++-- .../kubernetes/impl/KubernetesAccess.java | 153 +++++++------ .../io/oneko/metrics/MetricNameBuilder.java | 45 ++++ .../websocket/SessionWebSocketHandler.java | 19 +- src/main/resources/application.yaml | 10 +- src/test/java/io/oneko/InMemoryTestBench.java | 3 +- .../DockerRegistryInMemoryRepositoryTest.java | 3 +- .../io/oneko/event/EventDispatcherTest.java | 4 +- ...ndUtilsTest.java => HelmCommandsTest.java} | 23 +- .../EventAwareProjectRepositoryTest.java | 3 +- 26 files changed, 805 insertions(+), 362 deletions(-) create mode 100644 src/main/java/io/oneko/docker/v2/metrics/DockerRegistryClientMetrics.java create mode 100644 src/main/java/io/oneko/docker/v2/metrics/MetersPerRegistry.java create mode 100644 src/main/java/io/oneko/helm/HelmCommands.java delete mode 100644 src/main/java/io/oneko/helm/util/HelmCommandUtils.java create mode 100644 src/main/java/io/oneko/metrics/MetricNameBuilder.java rename src/test/java/io/oneko/helm/{util/HelmCommandUtilsTest.java => HelmCommandsTest.java} (63%) diff --git a/README.md b/README.md index e27a2eb0..8de815e4 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ development versions of your software into Kubernetes via Helm to allow everybod * You need Helm charts for each project you want to deploy. The charts need to be hosted in a chart registry. * Currently we support standard Helm chart registries and Helm GCS * The Docker image tag and the image pull policy need to be configurable -* O-Neko works with kubernetes versions 1.10.0 - 1.22.1 (these versions are officially supported by the Kubernetes client library we use) +* O-Neko works with kubernetes versions 1.10.0 - 1.23.3 (these versions are *officially* supported by the Kubernetes client library we use) ## How does it work? diff --git a/pom.xml b/pom.xml index 7b025916..7ead5481 100644 --- a/pom.xml +++ b/pom.xml @@ -23,11 +23,11 @@ UTF-8 UTF-8 - 2.4.1 - 10.11 - 1.4.1.Final - 1.18.16 - 1.5.10 + 2.7.0 + 11.8 + 1.5.2.Final + 1.18.24 + 1.6.9 @@ -69,6 +69,11 @@ org.springframework.boot spring-boot-starter-security + + io.micrometer + micrometer-registry-prometheus + runtime + io.github.openfeign feign-core @@ -115,7 +120,7 @@ com.github.ben-manes.caffeine caffeine - 2.8.8 + 3.1.1 org.projectlombok @@ -127,7 +132,7 @@ org.apache.commons commons-lang3 - 3.11 + 3.12.0 org.apache.commons @@ -142,34 +147,34 @@ commons-io commons-io - 2.8.0 + 2.11.0 com.google.code.gson gson - 2.8.9 + 2.9.0 compile org.yaml snakeyaml - 1.27 + 1.30 de.flapdoodle.embed de.flapdoodle.embed.mongo - 3.0.0 + 3.4.6 test com.google.guava guava - 30.1.1-jre + 31.1-jre io.fabric8 kubernetes-client - 5.8.0 + 5.12.2 org.mapstruct @@ -179,7 +184,7 @@ org.assertj assertj-core - 3.18.1 + 3.23.1 test @@ -190,7 +195,7 @@ net.logstash.logback logstash-logback-encoder - 6.6 + 7.2 org.springdoc diff --git a/src/main/java/io/oneko/automations/ScheduledLifetimeController.java b/src/main/java/io/oneko/automations/ScheduledLifetimeController.java index 69a217ff..aece0eea 100644 --- a/src/main/java/io/oneko/automations/ScheduledLifetimeController.java +++ b/src/main/java/io/oneko/automations/ScheduledLifetimeController.java @@ -1,8 +1,7 @@ package io.oneko.automations; -import static io.oneko.util.MoreStructuredArguments.projectKv; -import static io.oneko.util.MoreStructuredArguments.versionKv; -import static net.logstash.logback.argument.StructuredArguments.kv; +import static io.oneko.util.MoreStructuredArguments.*; +import static net.logstash.logback.argument.StructuredArguments.*; import java.util.List; import java.util.Optional; @@ -15,20 +14,21 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import io.oneko.kubernetes.DeploymentManager; import io.oneko.kubernetes.deployments.DeployableStatus; import io.oneko.kubernetes.deployments.Deployment; import io.oneko.kubernetes.deployments.DeploymentRepository; +import io.oneko.metrics.MetricNameBuilder; import io.oneko.project.ProjectRepository; import io.oneko.project.ProjectVersion; import io.oneko.project.ReadableProject; import io.oneko.project.WritableProjectVersion; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; @Component @Slf4j -@AllArgsConstructor public class ScheduledLifetimeController { private final LifetimeBehaviourService lifetimeBehaviourService; @@ -36,31 +36,64 @@ public class ScheduledLifetimeController { private final DeploymentRepository deploymentRepository; private final DeploymentManager deploymentManager; + private final Timer scheduledProjectCheckTimer; + private final Timer expiredDeploymentStopTimer; + private final Timer retrieveExpiredDeploymentsTimer; + + public ScheduledLifetimeController(LifetimeBehaviourService lifetimeBehaviourService, + ProjectRepository projectRepository, + DeploymentRepository deploymentRepository, + DeploymentManager deploymentManager, + MeterRegistry meterRegistry) { + this.lifetimeBehaviourService = lifetimeBehaviourService; + this.projectRepository = projectRepository; + this.deploymentRepository = deploymentRepository; + this.deploymentManager = deploymentManager; + + this.scheduledProjectCheckTimer = Timer.builder(new MetricNameBuilder().durationOf("lifetime.scheduled.checkProjects").build()) + .description("the time it takes O-Neko to check all projects for versions which have a lifetime configuration which needs to be checked") + .publishPercentileHistogram() + .register(meterRegistry); + this.retrieveExpiredDeploymentsTimer = Timer.builder(new MetricNameBuilder().durationOf("lifetime.scheduled.deployments.retrieveExpired").build()) + .description("the time it takes O-Neko to filter and retrieve expired deployments") + .publishPercentileHistogram() + .register(meterRegistry); + this.expiredDeploymentStopTimer = Timer.builder(new MetricNameBuilder().durationOf("lifetime.scheduled.deployments.stopExpired").build()) + .description("the time it takes O-Neko to stop an individual expired deployment") + .publishPercentileHistogram() + .register(meterRegistry); + } + @Scheduled(fixedRate = 5 * 60000) public void checkProjects() { - final List> versions = projectRepository.getAll().stream() + final var sample = Timer.start(); + final List> versions = projectRepository.getAll().stream() .map(ReadableProject::writable) .flatMap(project -> project.getVersions().stream()) .filter(this::shouldConsiderVersion) .collect(Collectors.toList()); - + sample.stop(scheduledProjectCheckTimer); stopExpiredDeployments(versions, projectVersion -> log.info("deployment expired ({}, {})", versionKv(projectVersion), projectKv(projectVersion.getProject()))); } - private void stopExpiredDeployments(List> deployables, Consumer> beforeStopDeployment) { + private void stopExpiredDeployments(List> deployables, Consumer> beforeStopDeployment) { + final Timer.Sample retrieveDeploymentsStart = Timer.start(); final var deployments = getRelevantDeploymentsFor(deployables); final var expiredPairsOfDeployableAndDeployment = getExpiredPairsOfDeployableAndDeployment(deployables, deployments); - - expiredPairsOfDeployableAndDeployment.forEach(expiredVersionDeploymentPair -> { - final var projectVersion = expiredVersionDeploymentPair.getLeft(); - beforeStopDeployment.accept(projectVersion); - if (projectVersion instanceof WritableProjectVersion) { - deploymentManager.stopDeployment((WritableProjectVersion) projectVersion); - } else { - log.error("stopping is not supported ({})", kv("class_name", projectVersion.getClass())); - } - }); + retrieveDeploymentsStart.stop(retrieveExpiredDeploymentsTimer); + + expiredPairsOfDeployableAndDeployment.forEach(expiredDeploymentStopTimer.record(() -> + expiredVersionDeploymentPair -> { + final var projectVersion = expiredVersionDeploymentPair.getLeft(); + beforeStopDeployment.accept(projectVersion); + if (projectVersion instanceof WritableProjectVersion) { + deploymentManager.stopDeployment((WritableProjectVersion) projectVersion); + } else { + log.error("stopping is not supported ({})", kv("class_name", projectVersion.getClass())); + } + }) + ); } private boolean shouldConsiderVersion(ProjectVersion version) { @@ -72,14 +105,14 @@ private boolean shouldConsider(Optional behaviour) { return behaviour.isPresent() && !behaviour.get().isInfinite(); } - private List getRelevantDeploymentsFor(List> deployables) { + private List getRelevantDeploymentsFor(List> deployables) { final var uuids = deployables.stream().map(ProjectVersion::getId).collect(Collectors.toSet()); return deploymentRepository.findAllByProjectVersionIdIn(uuids).stream() .filter(deployment -> !deployment.getStatus().equals(DeployableStatus.NotScheduled)) .collect(Collectors.toList()); } - private Set, Deployment>> getExpiredPairsOfDeployableAndDeployment(List> versions, List deployments) { + private Set, Deployment>> getExpiredPairsOfDeployableAndDeployment(List> versions, List deployments) { var combiningFunction = createExpiredDeployableDeploymentCombiningFunction(versions); return deployments.stream() .map(combiningFunction) @@ -89,7 +122,7 @@ private Set, Deployment>> getExpiredPairsOfDeployableAn } //what a method name - private Function, Deployment>>> createExpiredDeployableDeploymentCombiningFunction(List> deployables) { + private Function, Deployment>>> createExpiredDeployableDeploymentCombiningFunction(List> deployables) { return (deployment) -> { final var matchingDeployableOptional = deployables.stream() diff --git a/src/main/java/io/oneko/docker/DockerRegistryPolling.java b/src/main/java/io/oneko/docker/DockerRegistryPolling.java index 56f67683..3312b333 100644 --- a/src/main/java/io/oneko/docker/DockerRegistryPolling.java +++ b/src/main/java/io/oneko/docker/DockerRegistryPolling.java @@ -24,6 +24,8 @@ import com.google.common.collect.Sets; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import io.oneko.docker.event.NewProjectVersionFoundEvent; import io.oneko.docker.event.ObsoleteProjectVersionRemovedEvent; import io.oneko.docker.v2.DockerRegistryClientFactory; @@ -34,6 +36,7 @@ import io.oneko.event.EventTrigger; import io.oneko.event.ScheduledTask; import io.oneko.kubernetes.DeploymentManager; +import io.oneko.metrics.MetricNameBuilder; import io.oneko.project.ProjectRepository; import io.oneko.project.ProjectVersion; import io.oneko.project.ReadableProject; @@ -47,6 +50,8 @@ @Slf4j class DockerRegistryPolling { + + @Data private static class VersionWithDockerManifest { private final WritableProjectVersion version; @@ -63,18 +68,29 @@ private static class VersionWithDockerManifest { private final EventTrigger asTrigger; private final ExpiringBucket failedManifestRequests = new ExpiringBucket(Duration.ofMinutes(5)).concurrent(); private final CurrentEventTrigger currentEventTrigger; + private final Timer pollingJobTimer; + private final Timer updateDatesJobTimer; DockerRegistryPolling(ProjectRepository projectRepository, DockerRegistryClientFactory dockerRegistryClientFactory, DeploymentManager deploymentManager, EventDispatcher eventDispatcher, - CurrentEventTrigger currentEventTrigger) { + CurrentEventTrigger currentEventTrigger, + MeterRegistry meterRegistry) { this.projectRepository = projectRepository; this.dockerRegistryClientFactory = dockerRegistryClientFactory; this.deploymentManager = deploymentManager; this.eventDispatcher = eventDispatcher; this.currentEventTrigger = currentEventTrigger; this.asTrigger = new ScheduledTask("Docker Registry Polling"); + this.pollingJobTimer = Timer.builder(new MetricNameBuilder().durationOf("docker.registry.polling.pollingJob").build()) + .description("the duration of the docker polling job") + .publishPercentileHistogram() + .register(meterRegistry); + this.updateDatesJobTimer = Timer.builder(new MetricNameBuilder().durationOf("docker.registry.polling.updateDatesJob").build()) + .description("the duration of the image date update job") + .publishPercentileHistogram() + .register(meterRegistry); } @Scheduled(fixedDelay = 20000, initialDelay = 10000) @@ -99,6 +115,7 @@ protected void updateAndRedeployAllIfRequired() { } log.trace("finished polling job ({})", kv("duration_millis", stopWatch.getTime())); + pollingJobTimer.record(Duration.ofMillis(stopWatch.getTime())); } } @@ -121,6 +138,7 @@ protected void updateDatesForAllImagesAndAllTags() { } log.trace("finished updating dates for all projects ({})", kv("duration_millis", stopWatch.getTime())); + updateDatesJobTimer.record(Duration.ofMillis(stopWatch.getTime())); } /** diff --git a/src/main/java/io/oneko/docker/v2/DockerRegistryClientFactory.java b/src/main/java/io/oneko/docker/v2/DockerRegistryClientFactory.java index 59cb8ab3..8c2bfd73 100644 --- a/src/main/java/io/oneko/docker/v2/DockerRegistryClientFactory.java +++ b/src/main/java/io/oneko/docker/v2/DockerRegistryClientFactory.java @@ -21,11 +21,16 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.binder.cache.CaffeineCacheMetrics; import io.oneko.docker.DockerRegistry; import io.oneko.docker.DockerRegistryRepository; import io.oneko.docker.v2.DockerRegistryVersionChecker.BearerAuthRequired; import io.oneko.docker.v2.DockerRegistryVersionChecker.V2APIUnavailable; +import io.oneko.docker.v2.metrics.DockerRegistryClientMetrics; import io.oneko.docker.v2.model.TokenResponse; +import io.oneko.metrics.MetricNameBuilder; import io.oneko.project.Project; import lombok.extern.slf4j.Slf4j; @@ -36,15 +41,46 @@ public class DockerRegistryClientFactory { private final ObjectMapper objectMapper; private final DockerRegistryVersionChecker dockerRegistryVersionChecker; private final DockerRegistryRepository dockerRegistryRepository; + private final DockerRegistryClientMetrics dockerRegistryClientMetrics; private final Cache projectToDockerRegistryClientCache = Caffeine.newBuilder() .expireAfterWrite(Duration.ofMinutes(5)) + .recordStats() .build(); + private final Timer clientBuildTimerCatalog; + + private final Timer clientBuildTimerRepository; + + private final Timer tokenRequestTimer; + + @Autowired - DockerRegistryClientFactory(ObjectMapper objectMapper, DockerRegistryVersionChecker dockerRegistryVersionChecker, DockerRegistryRepository dockerRegistryRepository) { + DockerRegistryClientFactory(ObjectMapper objectMapper, + DockerRegistryVersionChecker dockerRegistryVersionChecker, + DockerRegistryRepository dockerRegistryRepository, + MeterRegistry meterRegistry, + DockerRegistryClientMetrics dockerRegistryClientMetrics) { + this.objectMapper = objectMapper; this.dockerRegistryVersionChecker = dockerRegistryVersionChecker; this.dockerRegistryRepository = dockerRegistryRepository; + this.dockerRegistryClientMetrics = dockerRegistryClientMetrics; + + CaffeineCacheMetrics.monitor(meterRegistry, projectToDockerRegistryClientCache, "dockerRegistryClientCache"); + clientBuildTimerCatalog = Timer.builder(new MetricNameBuilder().durationOf("docker.registry.client.build").build()) + .description("the time it takes to build a container registry client") + .tag("scope", "catalog") + .publishPercentileHistogram() + .register(meterRegistry); + clientBuildTimerRepository = Timer.builder(new MetricNameBuilder().durationOf("docker.registry.client.build").build()) + .description("the time it takes to build a container registry client") + .tag("scope", "repository") + .publishPercentileHistogram() + .register(meterRegistry); + tokenRequestTimer = Timer.builder(new MetricNameBuilder().durationOf("docker.registry.tokenrequest").build()) + .description("the time it takes to request the token for interacting with the registry") + .publishPercentileHistogram() + .register(meterRegistry); } /** @@ -59,8 +95,10 @@ public String checkRegistryAvailability(DockerRegistry dockerRegistry) { } public DockerRegistryV2Client getDockerRegistryClient(DockerRegistry dockerRegistry) { - var dockerRegistryCheckResult = dockerRegistryVersionChecker.checkV2ApiOf(dockerRegistry); - return buildClientBasedOnApiCheck(dockerRegistryCheckResult, dockerRegistry, "registry:catalog:*"); + return clientBuildTimerCatalog.record(() -> { + var dockerRegistryCheckResult = dockerRegistryVersionChecker.checkV2ApiOf(dockerRegistry); + return buildClientBasedOnApiCheck(dockerRegistryCheckResult, dockerRegistry, "registry:catalog:*"); + }); } public Optional getDockerRegistryClient(Project project) { @@ -71,11 +109,12 @@ public Optional getDockerRegistryClient(Project pr } private Optional buildDockerRegistryClientForProject(Project project) { - return dockerRegistryRepository.getById(project.getDockerRegistryId()) + return clientBuildTimerRepository.record(() -> dockerRegistryRepository.getById(project.getDockerRegistryId()) .map(dockerRegistry -> { DockerRegistryVersionChecker.DockerRegistryCheckResult dockerRegistryCheckResult = dockerRegistryVersionChecker.checkV2ApiOf(dockerRegistry); return this.buildClientBasedOnApiCheck(dockerRegistryCheckResult, dockerRegistry, "repository:" + project.getImageName() + ":pull"); - }); + }) + ); } /** @@ -90,13 +129,13 @@ private DockerRegistryV2Client buildClientBasedOnApiCheck(DockerRegistryVersionC if (checkResult == DockerRegistryVersionChecker.DockerRegistryCheckResult.UnsupportedAuthenticationType) { throw new IllegalStateException("Docker registry " + registry.getName() + " does not support the V2 API"); } else if (checkResult == DockerRegistryVersionChecker.DockerRegistryCheckResult.Okay) { - return new DockerRegistryV2Client(registry, null, objectMapper); + return new DockerRegistryV2Client(registry, null, objectMapper, dockerRegistryClientMetrics.getMeters(registry)); } else if (checkResult instanceof V2APIUnavailable) { throw new IllegalStateException("V2 API of docker registry " + registry.getName() + " is unavailable with status code " + ((V2APIUnavailable) checkResult).getStatusCode()); } else if (checkResult instanceof BearerAuthRequired) { BearerAuthRequired required = (BearerAuthRequired) checkResult; TokenResponse tokenResponse = requestToken(registry, required, desiredScope); - return new DockerRegistryV2Client(registry, tokenResponse.getToken(), objectMapper); + return new DockerRegistryV2Client(registry, tokenResponse.getToken(), objectMapper, dockerRegistryClientMetrics.getMeters(registry)); } else { throw new IllegalStateException("CheckResult" + checkResult + " not supporter by docker client factory"); } @@ -107,6 +146,7 @@ private DockerRegistryV2Client buildClientBasedOnApiCheck(DockerRegistryVersionC * basic authentication with the users credentials. */ private TokenResponse requestToken(DockerRegistry registry, BearerAuthRequired required, String scope) { + final Timer.Sample sample = Timer.start(); HttpClientBuilder builder = HttpClients.custom(); if (registry.isTrustInsecureCertificate()) { builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); @@ -117,7 +157,9 @@ private TokenResponse requestToken(DockerRegistry registry, BearerAuthRequired r request.addHeader(new BasicHeader(HttpHeaders.AUTHORIZATION, AuthorizationHeader.basic(registry.getUserName(), registry.getPassword()))); try (CloseableHttpResponse response = client.execute(request)) { - return this.objectMapper.readValue(EntityUtils.toString(response.getEntity()), TokenResponse.class); + final TokenResponse tokenResponse = this.objectMapper.readValue(EntityUtils.toString(response.getEntity()), TokenResponse.class); + sample.stop(tokenRequestTimer); + return tokenResponse; } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/io/oneko/docker/v2/DockerRegistryV2Client.java b/src/main/java/io/oneko/docker/v2/DockerRegistryV2Client.java index 1ea9149a..de25534e 100644 --- a/src/main/java/io/oneko/docker/v2/DockerRegistryV2Client.java +++ b/src/main/java/io/oneko/docker/v2/DockerRegistryV2Client.java @@ -21,7 +21,9 @@ import feign.jackson.JacksonDecoder; import feign.jackson.JacksonEncoder; import feign.slf4j.Slf4jLogger; +import io.micrometer.core.instrument.Timer; import io.oneko.docker.DockerRegistry; +import io.oneko.docker.v2.metrics.MetersPerRegistry; import io.oneko.docker.v2.model.manifest.DockerRegistryBlob; import io.oneko.docker.v2.model.manifest.DockerRegistryManifest; import io.oneko.docker.v2.model.manifest.Manifest; @@ -37,8 +39,13 @@ public class DockerRegistryV2Client { private final DockerRegistryAPIV2 feignClient; + private final MetersPerRegistry meters; - public DockerRegistryV2Client(DockerRegistry registry, String token, ObjectMapper objectMapper) { + public DockerRegistryV2Client(DockerRegistry registry, + String token, + ObjectMapper objectMapper, + MetersPerRegistry meters) { + this.meters = meters; List
defaultHeaders = new ArrayList<>(); defaultHeaders.add(new BasicHeader("Accept", "*/*")); if (token != null) { @@ -62,31 +69,42 @@ public DockerRegistryV2Client(DockerRegistry registry, String token, ObjectMappe } public String versionCheck() { + final Timer.Sample sample = Timer.start(); try { - return feignClient.versionCheck(); + final String result = feignClient.versionCheck(); + sample.stop(meters.getVersionCheckTimerOk()); + return result; } catch (FeignException e) { + sample.stop(meters.getVersionCheckTimerError()); log.warn("failed to check docker registry version", e); throw e; } } public List getAllTags(Project project) { + final Timer.Sample sample = Timer.start(); try { - return feignClient.getAllTags(project.getImageName()).getTags(); + final List result = feignClient.getAllTags(project.getImageName()).getTags(); + sample.stop(meters.getListAllTagsTimerOk()); + return result; } catch (FeignException e) { + sample.stop(meters.getListAllTagsTimerError()); log.warn("failed to list all container image tags ({})", kv("image_name", project.getImageName()), e); throw e; } } public Manifest getManifest(ProjectVersion version) { + final Timer.Sample sample = Timer.start(); try { final String imageName = version.getProject().getImageName(); final DockerRegistryManifest dockerRegistryManifest = feignClient.getManifest(imageName, version.getName()); final DockerRegistryManifest.Digest digest = dockerRegistryManifest.getDigest(); final DockerRegistryBlob blob = feignClient.getBlob(imageName, digest.getAlgorithm(), digest.getDigest()); + sample.stop(meters.getGetManifestTimerOk()); return new Manifest(digest.getFullDigest(), blob.getCreated()); } catch (FeignException e) { + sample.stop(meters.getGetManifestTimerError()); log.warn("failed to get manifest for project version ({}, {})", versionKv(version), projectKv(version.getProject()), e); throw e; } diff --git a/src/main/java/io/oneko/docker/v2/DockerRegistryVersionChecker.java b/src/main/java/io/oneko/docker/v2/DockerRegistryVersionChecker.java index 8dadae3b..ef466094 100644 --- a/src/main/java/io/oneko/docker/v2/DockerRegistryVersionChecker.java +++ b/src/main/java/io/oneko/docker/v2/DockerRegistryVersionChecker.java @@ -16,7 +16,10 @@ import org.apache.http.impl.client.HttpClients; import org.springframework.stereotype.Component; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import io.oneko.docker.DockerRegistry; +import io.oneko.metrics.MetricNameBuilder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -26,8 +29,24 @@ @Slf4j @Component public class DockerRegistryVersionChecker { + private final Timer versionCheckTimerSuccess; + private final Timer versionCheckTimerError; + + public DockerRegistryVersionChecker(MeterRegistry meterRegistry) { + versionCheckTimerSuccess = Timer.builder(new MetricNameBuilder().durationOf("docker.registry.versioncheck").build()) + .description("time it takes to check whether the v2 api of a container registry is available") + .publishPercentileHistogram() + .tag("success", "true") + .register(meterRegistry); + versionCheckTimerError = Timer.builder(new MetricNameBuilder().durationOf("docker.registry.versioncheck").build()) + .description("time it takes to check whether the v2 api of a container registry is available") + .publishPercentileHistogram() + .tag("success", "false") + .register(meterRegistry); + } public DockerRegistryCheckResult checkV2ApiOf(DockerRegistry registry) { + final Timer.Sample sample = Timer.start(); final var builder = HttpClients.custom(); if (registry.isTrustInsecureCertificate()) { builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); @@ -35,8 +54,11 @@ public DockerRegistryCheckResult checkV2ApiOf(DockerRegistry registry) { CloseableHttpClient client = builder.build(); HttpGet request = new HttpGet(registry.getRegistryUrl() + "/v2/"); try (CloseableHttpResponse response = client.execute(request)) { - return mapResponseToCheckResult(response); + final var result = mapResponseToCheckResult(response); + sample.stop(versionCheckTimerSuccess); + return result; } catch (IOException e) { + sample.stop(versionCheckTimerError); log.error("failed to check the docker V2 API ({})", containerRegistryKv(registry), e); throw new IllegalStateException(e); } diff --git a/src/main/java/io/oneko/docker/v2/metrics/DockerRegistryClientMetrics.java b/src/main/java/io/oneko/docker/v2/metrics/DockerRegistryClientMetrics.java new file mode 100644 index 00000000..211afacf --- /dev/null +++ b/src/main/java/io/oneko/docker/v2/metrics/DockerRegistryClientMetrics.java @@ -0,0 +1,28 @@ +package io.oneko.docker.v2.metrics; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.springframework.stereotype.Component; + +import io.micrometer.core.instrument.MeterRegistry; +import io.oneko.docker.DockerRegistry; +import lombok.Getter; + +@Component +@Getter +public class DockerRegistryClientMetrics { + private final MeterRegistry meterRegistry; + private static Map metersByRegistry = Collections.synchronizedMap(new HashMap<>()); + + DockerRegistryClientMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + public MetersPerRegistry getMeters(DockerRegistry dockerRegistry) { + return metersByRegistry.computeIfAbsent(dockerRegistry.getUuid(), ignored -> new MetersPerRegistry(dockerRegistry, meterRegistry)); + } + +} diff --git a/src/main/java/io/oneko/docker/v2/metrics/MetersPerRegistry.java b/src/main/java/io/oneko/docker/v2/metrics/MetersPerRegistry.java new file mode 100644 index 00000000..194866e5 --- /dev/null +++ b/src/main/java/io/oneko/docker/v2/metrics/MetersPerRegistry.java @@ -0,0 +1,58 @@ +package io.oneko.docker.v2.metrics; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.oneko.docker.DockerRegistry; +import io.oneko.metrics.MetricNameBuilder; +import lombok.Getter; + +@Getter +public class MetersPerRegistry { + private final Timer versionCheckTimerOk; + private final Timer versionCheckTimerError; + + private final Timer listAllTagsTimerOk; + private final Timer listAllTagsTimerError; + + private final Timer getManifestTimerOk; + private final Timer getManifestTimerError; + + public MetersPerRegistry(DockerRegistry dockerRegistry, MeterRegistry meterRegistry) { + versionCheckTimerOk = builder(dockerRegistry) + .tag("operation", "versionCheck") + .tag("result", "success") + .register(meterRegistry); + + versionCheckTimerError = builder(dockerRegistry) + .tag("operation", "versionCheck") + .tag("result", "error") + .register(meterRegistry); + + listAllTagsTimerOk = builder(dockerRegistry) + .tag("operation", "listAllTags") + .tag("result", "success") + .register(meterRegistry); + + listAllTagsTimerError = builder(dockerRegistry) + .tag("operation", "listAllTags") + .tag("result", "error") + .register(meterRegistry); + + getManifestTimerOk = builder(dockerRegistry) + .tag("operation", "getManifest") + .tag("result", "success") + .register(meterRegistry); + + getManifestTimerError = builder(dockerRegistry) + .tag("operation", "getManifest") + .tag("result", "error") + .register(meterRegistry); + } + + private static Timer.Builder builder(DockerRegistry dockerRegistry) { + return Timer.builder(new MetricNameBuilder().durationOf("docker.registry.client.request").build()) + .description("the time it takes to make requests to the container registry") + .publishPercentileHistogram() + .tag("registry", dockerRegistry.getName()); + } +} diff --git a/src/main/java/io/oneko/event/EventDispatcher.java b/src/main/java/io/oneko/event/EventDispatcher.java index 2e6e8bf5..212ec682 100644 --- a/src/main/java/io/oneko/event/EventDispatcher.java +++ b/src/main/java/io/oneko/event/EventDispatcher.java @@ -6,6 +6,10 @@ import org.springframework.stereotype.Service; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.oneko.metrics.MetricNameBuilder; + @Service public class EventDispatcher { @@ -13,8 +17,13 @@ public class EventDispatcher { private final Set> listeners = new HashSet<>(); private final CurrentEventTrigger currentEventTrigger; - public EventDispatcher(CurrentEventTrigger currentEventTrigger) { + private final Counter eventsCounter; + + public EventDispatcher(CurrentEventTrigger currentEventTrigger, MeterRegistry meterRegistry) { this.currentEventTrigger = currentEventTrigger; + eventsCounter = Counter.builder(new MetricNameBuilder().amountOf("events.dispatched").build()) + .description("the number of events dispatched by the EventDispatcher since application start") + .register(meterRegistry); } public void registerListener(Consumer listener) { @@ -26,10 +35,10 @@ public void removeListener(Consumer listener) { } public void dispatch(Event event) { + eventsCounter.increment(); if (event.getTrigger() == null) { event.setTrigger(currentEventTrigger.currentTrigger().orElse(UnknownTrigger.INSTANCE)); } - listeners.forEach(listener -> listener.accept(event)); } diff --git a/src/main/java/io/oneko/helm/HelmCharts.java b/src/main/java/io/oneko/helm/HelmCharts.java index 37c50888..49b4a10a 100644 --- a/src/main/java/io/oneko/helm/HelmCharts.java +++ b/src/main/java/io/oneko/helm/HelmCharts.java @@ -8,54 +8,48 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.springframework.stereotype.Component; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; -import io.oneko.helm.util.HelmCommandUtils; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.cache.CaffeineCacheMetrics; import io.oneko.helmapi.model.Chart; import lombok.extern.slf4j.Slf4j; @Component @Slf4j public class HelmCharts { - - private final HelmRegistryRepository helmRegistryRepository; - - private final LoadingCache chartsCache = CacheBuilder.newBuilder() - .expireAfterWrite(3, TimeUnit.MINUTES) - .build(new CacheLoader<>() { - @Override - public HelmChartsDTO load(UUID registryId) { - return helmRegistryRepository.getById(registryId) - .flatMap(helmRegistry -> { - try { - List charts = HelmCommandUtils.getCharts(helmRegistry); - log.debug("found helm charts ({}, {})", kv("chart_count", charts.size()), helmRegistryKv(helmRegistry)); - - return toHelmChartDTO(helmRegistry, charts); - } catch (HelmRegistryException e) { - return Optional.empty(); - } - }) - .orElseThrow(() -> new IllegalArgumentException("registry not found")); - } - }); - - public HelmCharts(HelmRegistryRepository helmRegistryRepository) { - this.helmRegistryRepository = helmRegistryRepository; + private final LoadingCache chartsCache; + + public HelmCharts(HelmRegistryRepository helmRegistryRepository, HelmCommands helmCommands, MeterRegistry meterRegistry) { + this.chartsCache = Caffeine.newBuilder() + .expireAfterWrite(3, TimeUnit.MINUTES) + .recordStats() + .build(registryId -> helmRegistryRepository.getById(registryId) + .flatMap(helmRegistry -> { + try { + List charts = helmCommands.getCharts(helmRegistry); + log.debug("found helm charts ({}, {})", kv("chart_count", charts.size()), helmRegistryKv(helmRegistry)); + + return toHelmChartDTO(helmRegistry, charts); + } catch (HelmRegistryException e) { + return Optional.empty(); + } + }) + .orElseThrow(() -> new IllegalArgumentException("registry not found")) + ); + CaffeineCacheMetrics.monitor(meterRegistry, chartsCache, "helmChartCache"); } public Optional getChartsByHelmRegistry(UUID registryId) { try { return Optional.of(chartsCache.get(registryId)); - } catch (ExecutionException e) { + } catch (RuntimeException e) { return Optional.empty(); } } diff --git a/src/main/java/io/oneko/helm/HelmCommands.java b/src/main/java/io/oneko/helm/HelmCommands.java new file mode 100644 index 00000000..844932a5 --- /dev/null +++ b/src/main/java/io/oneko/helm/HelmCommands.java @@ -0,0 +1,207 @@ +package io.oneko.helm; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import com.google.common.annotations.VisibleForTesting; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.oneko.helmapi.api.Helm; +import io.oneko.helmapi.model.Chart; +import io.oneko.helmapi.model.InstallStatus; +import io.oneko.helmapi.model.Release; +import io.oneko.helmapi.model.Status; +import io.oneko.helmapi.model.Values; +import io.oneko.helmapi.process.CommandException; +import io.oneko.metrics.MetricNameBuilder; +import io.oneko.project.ProjectVersion; +import io.oneko.templates.WritableConfigurationTemplate; +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class HelmCommands { + + private final Helm helm = new Helm(); + + private final Counter commandErrorCounter; + private final Timer addRepoTimer; + private final Timer deleteRepoTimer; + private final Timer repoUpdateTimer; + private final Timer searchRepoTimer; + private final Timer installTimer; + private final Timer uninstallTimer; + private final Timer statusTimer; + + private Instant lastRepoUpdate = Instant.MIN; + + + public HelmCommands(MeterRegistry meterRegistry) { + this.commandErrorCounter = Counter.builder(new MetricNameBuilder().amountOf("helm.command.errors").build()) + .description("number of errors during Helm command execution") + .register(meterRegistry); + this.addRepoTimer = timer("repo_add", meterRegistry); + this.deleteRepoTimer = timer("repo_delete", meterRegistry); + this.repoUpdateTimer = timer("repo_update", meterRegistry); + this.searchRepoTimer = timer("search_repo", meterRegistry); + this.installTimer = timer("install", meterRegistry); + this.uninstallTimer = timer("uninstall", meterRegistry); + this.statusTimer = timer("status", meterRegistry); + } + + private Timer timer(String operation, MeterRegistry meterRegistry) { + return Timer.builder(new MetricNameBuilder().durationOf("helm.command").build()) + .description("duration of Helm commands") + .publishPercentileHistogram() + .tag("operation", operation) + .register(meterRegistry); + } + + public void addRegistry(ReadableHelmRegistry helmRegistry) throws HelmRegistryException { + try { + addRepoTimer.record(() -> + helm.addRepo(helmRegistry.getName(), helmRegistry.getUrl(), helmRegistry.getUsername(), helmRegistry.getPassword()) + ); + } catch (CommandException e) { + commandErrorCounter.increment(); + throw HelmRegistryException.fromCommandException(e, helmRegistry.getUrl(), helmRegistry.getName()); + } + } + + public void deleteRegistry(ReadableHelmRegistry helmRegistry) throws HelmRegistryException { + try { + deleteRepoTimer.record(() -> + helm.removeRepo(helmRegistry.getName()) + ); + } catch (CommandException e) { + commandErrorCounter.increment(); + throw HelmRegistryException.fromCommandException(e, helmRegistry.getUrl(), helmRegistry.getName()); + } + } + + public synchronized void updateReposNotTooOften() { + final Instant now = Instant.now(); + if (now.isBefore(lastRepoUpdate.plusSeconds(30))) { + log.debug("not updating helm repos because the last update was less than 30 seconds ago"); + return; + } + lastRepoUpdate = now; + repoUpdateTimer.record(helm::updateRepos); + } + + public List getCharts(ReadableHelmRegistry helmRegistry) throws HelmRegistryException { + try { + updateReposNotTooOften(); + return searchRepoTimer.record(() -> + helm.searchRepo(helmRegistry.getName() + "/", true, false) + ); + } catch (CommandException e) { + commandErrorCounter.increment(); + throw HelmRegistryException.fromCommandException(e, helmRegistry.getUrl(), helmRegistry.getName()); + } + } + + public List install(ProjectVersion projectVersion) throws HelmRegistryException { + try { + updateReposNotTooOften(); + final var sample = Timer.start(); + List calculatedConfigurationTemplates = projectVersion.getCalculatedConfigurationTemplates(); + List result = new ArrayList<>(); + for (int i = 0; i < calculatedConfigurationTemplates.size(); i++) { + WritableConfigurationTemplate template = calculatedConfigurationTemplates.get(i); + result.add(helm.install(getReleaseName(projectVersion, i), template.getChartName(), template.getChartVersion(), Values.fromYamlString(template.getContent()), projectVersion.getNamespaceOrElseFromProject(), false)); + } + sample.stop(installTimer); + return result; + } catch (CommandException e) { + commandErrorCounter.increment(); + throw new HelmRegistryException(e.getMessage()); + } + } + + public void uninstall(List releaseNames) throws HelmRegistryException { + try { + uninstallTimer.record(() -> + helm.listInAllNamespaces().stream() + .filter(release -> releaseNames.contains(release.getName())) + .forEach(release -> helm.uninstall(release.getName(), release.getNamespace())) + ); + } catch (CommandException e) { + commandErrorCounter.increment(); + throw new HelmRegistryException(e.getMessage()); + } + } + + public void uninstall(ProjectVersion projectVersion) throws HelmRegistryException { + try { + uninstallTimer.record(() -> + helm.listInAllNamespaces().stream() + .filter(release -> release.getName().startsWith(getReleaseNamePrefix(projectVersion))) + .forEach(release -> helm.uninstall(release.getName(), release.getNamespace())) + ); + } catch (CommandException e) { + commandErrorCounter.increment(); + throw new HelmRegistryException(e.getMessage()); + } + } + + public List status(ProjectVersion projectVersion) throws HelmRegistryException { + try { + return statusTimer.record(() -> + helm.listInAllNamespaces() + .stream() + .filter(release -> release.getName().startsWith(getReleaseNamePrefix(projectVersion))) + .map(release -> helm.status(release.getName(), release.getNamespace())) + .collect(Collectors.toList()) + ); + } catch (CommandException e) { + commandErrorCounter.increment(); + throw new HelmRegistryException(e.getMessage()); + } + } + + private String getReleaseName(ProjectVersion projectVersion, int templateIndex) { + final String fullReleaseName = getReleaseNamePrefix(projectVersion) + "-" + templateIndex + "-" + maxLength(Long.toString(System.currentTimeMillis()), 10); + return sanitizeReleaseName(maxLength(fullReleaseName, 53)); + } + + public List getReferencedHelmReleases(ProjectVersion projectVersion) { + final String namespace = projectVersion.getNamespaceOrElseFromProject(); + final List list = helm.list(namespace, null); + return list.stream() + .map(Release::getName) + .filter(name -> name.startsWith(getReleaseNamePrefix(projectVersion))) + .collect(Collectors.toList()); + } + + @VisibleForTesting + protected String getReleaseNamePrefix(ProjectVersion projectVersion) { + var projectName = maxLength(projectVersion.getProject().getName(), 10); + var versionName = maxLength(projectVersion.getName(), 18); + var versionId = maxLength(projectVersion.getId().toString(), 8); + return sanitizeReleaseName(String.format("%s-%s-%s", projectName, versionName, versionId)); + } + + private String sanitizeReleaseName(String in) { + String candidate = in.toLowerCase(); + candidate = candidate.replaceAll("_", "-"); + candidate = candidate.replaceAll("[^a-z0-9\\-]", StringUtils.EMPTY);//remove invalid chars (only alphanumeric and dash allowed) + candidate = candidate.replaceAll("^[\\-]*", StringUtils.EMPTY);//remove invalid start (remove dot, dash and underscore from start) + candidate = candidate.replaceAll("[\\-]*$", StringUtils.EMPTY);//remove invalid end (remove dot, dash and underscore from end) + if (StringUtils.isBlank(candidate)) { + throw new IllegalArgumentException("can not create a legal namespace name from " + in); + } + return candidate; + } + + private String maxLength(String in, int length) { + return in.substring(0, Math.min(in.length(), length)); + } +} diff --git a/src/main/java/io/oneko/helm/HelmRegistryInitializer.java b/src/main/java/io/oneko/helm/HelmRegistryInitializer.java index 4a506641..1b774cd3 100644 --- a/src/main/java/io/oneko/helm/HelmRegistryInitializer.java +++ b/src/main/java/io/oneko/helm/HelmRegistryInitializer.java @@ -1,29 +1,29 @@ package io.oneko.helm; import static io.oneko.util.MoreStructuredArguments.*; -import static net.logstash.logback.argument.StructuredArguments.*; import javax.annotation.PostConstruct; import org.springframework.stereotype.Service; -import io.oneko.helm.util.HelmCommandUtils; import lombok.extern.slf4j.Slf4j; @Service @Slf4j public class HelmRegistryInitializer { private final HelmRegistryRepository helmRegistryRepository; + private final HelmCommands helmCommands; - public HelmRegistryInitializer(HelmRegistryRepository helmRegistryRepository) { + public HelmRegistryInitializer(HelmRegistryRepository helmRegistryRepository, HelmCommands helmCommands) { this.helmRegistryRepository = helmRegistryRepository; + this.helmCommands = helmCommands; } @PostConstruct public void startup() { helmRegistryRepository.getAll().forEach(registry -> { try { - HelmCommandUtils.addRegistry(registry); + helmCommands.addRegistry(registry); log.info("helm registry successfully added ({})", helmRegistryKv(registry)); } catch (HelmRegistryException e) { log.error("error while adding helm registry during initialization ({})", helmRegistryKv(registry), e); diff --git a/src/main/java/io/oneko/helm/rest/HelmRegistryController.java b/src/main/java/io/oneko/helm/rest/HelmRegistryController.java index ca98193e..fb4d23bb 100644 --- a/src/main/java/io/oneko/helm/rest/HelmRegistryController.java +++ b/src/main/java/io/oneko/helm/rest/HelmRegistryController.java @@ -18,12 +18,12 @@ import io.oneko.configuration.Controllers; import io.oneko.helm.HelmCharts; import io.oneko.helm.HelmChartsDTO; +import io.oneko.helm.HelmCommands; import io.oneko.helm.HelmRegistryException; import io.oneko.helm.HelmRegistryMapper; import io.oneko.helm.HelmRegistryRepository; import io.oneko.helm.ReadableHelmRegistry; import io.oneko.helm.WritableHelmRegistry; -import io.oneko.helm.util.HelmCommandUtils; import io.oneko.project.ProjectRepository; import io.oneko.project.ReadableProject; import lombok.extern.slf4j.Slf4j; @@ -37,15 +37,18 @@ public class HelmRegistryController { private final ProjectRepository projectRepository; private final HelmCharts helmCharts; private final HelmRegistryMapper mapper; + private final HelmCommands helmCommands; public HelmRegistryController(HelmRegistryRepository helmRegistryRepository, ProjectRepository projectRepository, HelmCharts helmCharts, - HelmRegistryMapper mapper) { + HelmRegistryMapper mapper, + HelmCommands helmCommands) { this.helmRegistryRepository = helmRegistryRepository; this.projectRepository = projectRepository; this.helmCharts = helmCharts; this.mapper = mapper; + this.helmCommands = helmCommands; } @PreAuthorize("hasAnyRole('ADMIN', 'DOER')") @@ -60,7 +63,7 @@ List getAllRegistries() { @PostMapping HelmRegistryDTO createRegistry(@RequestBody CreateHelmRegistryDTO dto) throws HelmRegistryException { WritableHelmRegistry registry = mapper.createRegistryFromDTO(dto); - HelmCommandUtils.addRegistry(registry.readable()); + helmCommands.addRegistry(registry.readable()); final ReadableHelmRegistry persistedRegistry = helmRegistryRepository.add(registry); helmCharts.refreshHelmChartsInRegistry(persistedRegistry.getId()); @@ -78,7 +81,7 @@ HelmRegistryDTO getRegistryById(@PathVariable UUID id) { @PostMapping("/{id}") HelmRegistryDTO updateRegistry(@PathVariable UUID id, @RequestBody HelmRegistryDTO dto) throws HelmRegistryException { ReadableHelmRegistry registry = getRegistryOr404(id); - HelmCommandUtils.addRegistry(registry); + helmCommands.addRegistry(registry); WritableHelmRegistry updatedRegistry = mapper.updateRegistryFromDTO(registry.writable(), dto); ReadableHelmRegistry persistedReg = helmRegistryRepository.add(updatedRegistry); @@ -95,7 +98,7 @@ void deleteRegistry(@PathVariable UUID id) throws HelmRegistryException { throw new HelmRegistryException("The Helm registry is still referenced in projects"); } - HelmCommandUtils.deleteRegistry(registry); + helmCommands.deleteRegistry(registry); helmCharts.invalidateHelmChartsInRegistry(id); helmRegistryRepository.remove(registry); @@ -105,7 +108,7 @@ void deleteRegistry(@PathVariable UUID id) throws HelmRegistryException { @PostMapping("/{id}/password") HelmRegistryDTO changeRegistryPassword(@PathVariable UUID id, @RequestBody ChangeHelmRegistryPasswordDTO dto) throws HelmRegistryException { ReadableHelmRegistry registry = getRegistryOr404(id); - HelmCommandUtils.addRegistry(registry); + helmCommands.addRegistry(registry); WritableHelmRegistry writable = registry.writable(); writable.setPassword(dto.getPassword()); diff --git a/src/main/java/io/oneko/helm/util/HelmCommandUtils.java b/src/main/java/io/oneko/helm/util/HelmCommandUtils.java deleted file mode 100644 index b9a9ead7..00000000 --- a/src/main/java/io/oneko/helm/util/HelmCommandUtils.java +++ /dev/null @@ -1,153 +0,0 @@ -package io.oneko.helm.util; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import org.apache.commons.lang3.StringUtils; - -import com.google.common.annotations.VisibleForTesting; - -import io.oneko.helm.HelmRegistryException; -import io.oneko.helm.ReadableHelmRegistry; -import io.oneko.helmapi.api.Helm; -import io.oneko.helmapi.model.Chart; -import io.oneko.helmapi.model.InstallStatus; -import io.oneko.helmapi.model.Release; -import io.oneko.helmapi.model.Status; -import io.oneko.helmapi.model.Values; -import io.oneko.helmapi.process.CommandException; -import io.oneko.project.ProjectVersion; -import io.oneko.templates.WritableConfigurationTemplate; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; - -@UtilityClass -@Slf4j -public class HelmCommandUtils { - - private static final Helm helm = new Helm(); - private static Instant lastRepoUpdate = Instant.MIN; - - public static void addRegistry(ReadableHelmRegistry helmRegistry) throws HelmRegistryException { - try { - helm.addRepo(helmRegistry.getName(), helmRegistry.getUrl(), helmRegistry.getUsername(), helmRegistry.getPassword()); - } catch (CommandException e) { - throw HelmRegistryException.fromCommandException(e, helmRegistry.getUrl(), helmRegistry.getName()); - } - } - - public static void deleteRegistry(ReadableHelmRegistry helmRegistry) throws HelmRegistryException { - try { - helm.removeRepo(helmRegistry.getName()); - } catch (CommandException e) { - throw HelmRegistryException.fromCommandException(e, helmRegistry.getUrl(), helmRegistry.getName()); - } - } - - public static synchronized void updateReposNotTooOften() { - final Instant now = Instant.now(); - if (now.isBefore(lastRepoUpdate.plusSeconds(30))) { - log.debug("not updating helm repos because the last update was less than 30 seconds ago"); - return; - } - lastRepoUpdate = now; - helm.updateRepos(); - } - - public static List getCharts(ReadableHelmRegistry helmRegistry) throws HelmRegistryException { - try { - updateReposNotTooOften(); - return helm.searchRepo(helmRegistry.getName() + "/", true, false); - } catch (CommandException e) { - throw HelmRegistryException.fromCommandException(e, helmRegistry.getUrl(), helmRegistry.getName()); - } - } - - public static List install(ProjectVersion projectVersion) throws HelmRegistryException { - try { - updateReposNotTooOften(); - List calculatedConfigurationTemplates = projectVersion.getCalculatedConfigurationTemplates(); - List result = new ArrayList<>(); - for (int i = 0; i < calculatedConfigurationTemplates.size(); i++) { - WritableConfigurationTemplate template = calculatedConfigurationTemplates.get(i); - result.add(helm.install(getReleaseName(projectVersion, i), template.getChartName(), template.getChartVersion(), Values.fromYamlString(template.getContent()), projectVersion.getNamespaceOrElseFromProject(), false)); - } - return result; - } catch (CommandException e) { - throw new HelmRegistryException(e.getMessage()); - } - } - - public static void uninstall(List releaseNames) throws HelmRegistryException { - try { - helm.listInAllNamespaces().stream() - .filter(release -> releaseNames.contains(release.getName())) - .forEach(release -> helm.uninstall(release.getName(), release.getNamespace())); - } catch (CommandException e) { - throw new HelmRegistryException(e.getMessage()); - } - } - - public static void uninstall(ProjectVersion projectVersion) throws HelmRegistryException { - try { - helm.listInAllNamespaces().stream() - .filter(release -> release.getName().startsWith(getReleaseNamePrefix(projectVersion))) - .forEach(release -> helm.uninstall(release.getName(), release.getNamespace())); - } catch (CommandException e) { - throw new HelmRegistryException(e.getMessage()); - } - } - - public static List status(ProjectVersion projectVersion) throws HelmRegistryException { - try { - return helm.listInAllNamespaces() - .stream() - .filter(release -> release.getName().startsWith(getReleaseNamePrefix(projectVersion))) - .map(release -> helm.status(release.getName(), release.getNamespace())) - .collect(Collectors.toList()); - } catch (CommandException e) { - throw new HelmRegistryException(e.getMessage()); - } - } - - public List getReferencedHelmReleases(ProjectVersion projectVersion) { - final String namespace = projectVersion.getNamespaceOrElseFromProject(); - final List list = helm.list(namespace, null); - return list.stream() - .map(Release::getName) - .filter(name -> name.startsWith(getReleaseNamePrefix(projectVersion))) - .collect(Collectors.toList()); - } - - private static String getReleaseName(ProjectVersion projectVersion, int templateIndex) { - final String fullReleaseName = getReleaseNamePrefix(projectVersion) + "-" + templateIndex + "-" + maxLength(Long.toString(System.currentTimeMillis()), 10); - return sanitizeReleaseName(maxLength(fullReleaseName, 53)); - } - - @VisibleForTesting - protected static String getReleaseNamePrefix(ProjectVersion projectVersion) { - var projectName = maxLength(projectVersion.getProject().getName(), 10); - var projectId = maxLength(projectVersion.getProject().getId().toString(), 8); - var versionName = maxLength(projectVersion.getName(), 10); - var versionId = maxLength(projectVersion.getId().toString(), 8); - return sanitizeReleaseName(String.format("%s%s-%s%s", projectName, projectId, versionName, versionId)); - } - - private static String sanitizeReleaseName(String in) { - String candidate = in.toLowerCase(); - candidate = candidate.replaceAll("_", "-"); - candidate = candidate.replaceAll("[^a-z0-9\\-]", StringUtils.EMPTY);//remove invalid chars (only alphanumeric and dash allowed) - candidate = candidate.replaceAll("^[\\-]*", StringUtils.EMPTY);//remove invalid start (remove dot, dash and underscore from start) - candidate = candidate.replaceAll("[\\-]*$", StringUtils.EMPTY);//remove invalid end (remove dot, dash and underscore from end) - if (StringUtils.isBlank(candidate)) { - throw new IllegalArgumentException("can not create a legal namespace name from " + in); - } - return candidate; - } - - private static String maxLength(String in, int length) { - return in.substring(0, Math.min(in.length(), length)); - } -} diff --git a/src/main/java/io/oneko/kubernetes/impl/DeploymentManagerImpl.java b/src/main/java/io/oneko/kubernetes/impl/DeploymentManagerImpl.java index 5b094d09..2825d95f 100644 --- a/src/main/java/io/oneko/kubernetes/impl/DeploymentManagerImpl.java +++ b/src/main/java/io/oneko/kubernetes/impl/DeploymentManagerImpl.java @@ -15,6 +15,9 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import io.oneko.docker.event.ObsoleteProjectVersionRemovedEvent; import io.oneko.docker.v2.DockerRegistryClientFactory; import io.oneko.docker.v2.model.manifest.Manifest; @@ -22,14 +25,15 @@ import io.oneko.event.Event; import io.oneko.event.EventDispatcher; import io.oneko.event.HelmReleasesInstallEvent; +import io.oneko.helm.HelmCommands; import io.oneko.helm.HelmRegistryException; -import io.oneko.helm.util.HelmCommandUtils; import io.oneko.helmapi.model.InstallStatus; import io.oneko.helmapi.model.Status; import io.oneko.kubernetes.DeploymentManager; import io.oneko.kubernetes.deployments.DeploymentRepository; import io.oneko.kubernetes.deployments.ReadableDeployment; import io.oneko.kubernetes.deployments.WritableDeployment; +import io.oneko.metrics.MetricNameBuilder; import io.oneko.project.ProjectRepository; import io.oneko.project.ProjectVersion; import io.oneko.project.ProjectVersionLock; @@ -47,22 +51,51 @@ class DeploymentManagerImpl implements DeploymentManager { private final DeploymentRepository deploymentRepository; private final ProjectVersionLock projectVersionLock; private final EventDispatcher eventDispatcher; + private final HelmCommands helmCommands; + + private final Timer deployDurationTimer; + private final Timer stopDeploymentDurationTimer; + private final Counter startDeploymentErrors; + private final Counter stopDeploymentErrors; DeploymentManagerImpl(DockerRegistryClientFactory dockerRegistryClientFactory, ProjectRepository projectRepository, DeploymentRepository deploymentRepository, - EventDispatcher eventDispatcher, ProjectVersionLock projectVersionLock) { + EventDispatcher eventDispatcher, + ProjectVersionLock projectVersionLock, + HelmCommands helmCommands, + MeterRegistry meterRegistry) { this.dockerRegistryClientFactory = dockerRegistryClientFactory; this.projectRepository = projectRepository; this.deploymentRepository = deploymentRepository; this.projectVersionLock = projectVersionLock; this.eventDispatcher = eventDispatcher; + this.helmCommands = helmCommands; eventDispatcher.registerListener(this::consumeDeletedVersionEvent); + + deployDurationTimer = Timer.builder(new MetricNameBuilder().durationOf("kubernetes.deployment.action").build()) + .tag("action", "start") + .publishPercentileHistogram() + .register(meterRegistry); + + stopDeploymentDurationTimer = Timer.builder(new MetricNameBuilder().durationOf("kubernetes.deployment.action").build()) + .tag("action", "stop") + .publishPercentileHistogram() + .register(meterRegistry); + + startDeploymentErrors = Counter.builder(new MetricNameBuilder().amountOf("kubernetes.deployment.errors").build()) + .tag("action", "start") + .register(meterRegistry); + + stopDeploymentErrors = Counter.builder(new MetricNameBuilder().amountOf("kubernetes.deployment.errors").build()) + .tag("action", "stop") + .register(meterRegistry); } @Override public ReadableProjectVersion deploy(final WritableProjectVersion version) { + final Timer.Sample sample = Timer.start(); if (StringUtils.isBlank(version.getNamespaceOrElseFromProject())) { throw new RuntimeException("A namespace must be configured in the project."); } @@ -74,10 +107,10 @@ public ReadableProjectVersion deploy(final WritableProjectVersion version) { final WritableDeployment deployment = getOrCreateDeploymentForVersion(version); if (!deployment.getReleaseNames().isEmpty()) { - HelmCommandUtils.uninstall(deployment.getReleaseNames()); + helmCommands.uninstall(deployment.getReleaseNames()); } - final List installStatuses = HelmCommandUtils.install(version); + final List installStatuses = helmCommands.install(version); log.info("installing helm releases ({}, {})", kv("helm_releases", deployment.getReleaseNames()), versionKv(version)); @@ -87,17 +120,19 @@ public ReadableProjectVersion deploy(final WritableProjectVersion version) { eventDispatcher.dispatch(new HelmReleasesInstallEvent(version, releaseNames)); - return updateDeployableWithCreatedResources(version).map(newVersion -> { + final ReadableProjectVersion readableProjectVersion = updateDeployableWithCreatedResources(version).map(newVersion -> { newVersion.setDesiredState(Deployed); - final ReadableProject project = projectRepository.add(newVersion.getProject()); return project.getVersions().stream() .filter(projectVersion -> projectVersion.getUuid().equals(versionId)) .findFirst() .orElse(null); }).orElseThrow(() -> new RuntimeException("failed to update deployment from new version")); + sample.stop(deployDurationTimer); + return readableProjectVersion; } catch (Exception e) { log.error("failed to deploy ({})", versionKv(version), e); + startDeploymentErrors.increment(); rollback(version, e); throw new RuntimeException(e); } @@ -109,7 +144,7 @@ private void rollback(WritableProjectVersion version, Exception e) { try { eventDispatcher.dispatch(new DeploymentRollbackEvent(version, e.getMessage())); final WritableDeployment deployment = getOrCreateDeploymentForVersion(version); - final List referencedHelmReleases = HelmCommandUtils.getReferencedHelmReleases(version); + final List referencedHelmReleases = helmCommands.getReferencedHelmReleases(version); log.info("Found these helm releases for rollback: {}", kv("helm_releases", referencedHelmReleases)); if (!referencedHelmReleases.isEmpty()) { @@ -118,12 +153,13 @@ private void rollback(WritableProjectVersion version, Exception e) { if (!CollectionUtils.isEqualCollection(deployment.getReleaseNames(), referencedHelmReleases)) { log.warn("Orphaned helm release for project version {} detected. It will be removed.", versionKv(version)); } - HelmCommandUtils.uninstall(referencedHelmReleases); + helmCommands.uninstall(referencedHelmReleases); deployment.setReleaseNames(new ArrayList<>()); deploymentRepository.save(deployment); } } catch (Exception e2) { log.error("rollback deployment of {} failed", versionKv(version) , e2); + stopDeploymentErrors.increment(); throw new RuntimeException(e); } } @@ -150,10 +186,11 @@ private Optional updateDeployableWithCreatedResources(Wr @Override public ReadableProjectVersion stopDeployment(final WritableProjectVersion version) { + final Timer.Sample sample = Timer.start(); return projectVersionLock.doWithProjectVersionLock(version, () -> { try { final WritableDeployment deployment = getOrCreateDeploymentForVersion(version); - HelmCommandUtils.uninstall(version); + helmCommands.uninstall(version); deploymentRepository.deleteById(deployment.getId()); version.setDesiredState(NotDeployed); final ReadableProject readableProject = projectRepository.add(version.getProject()); @@ -161,23 +198,29 @@ public ReadableProjectVersion stopDeployment(final WritableProjectVersion versio log.info("stopping helm releases ({}, {})", kv("helm_releases", deployment.getReleaseNames()), versionKv(version)); - return readableProject.getVersions().stream() + final ReadableProjectVersion readableProjectVersion = readableProject.getVersions().stream() .filter(projectVersion -> projectVersion.getUuid().equals(version.getId())) .findFirst().orElse(null); + sample.stop(stopDeploymentDurationTimer); + return readableProjectVersion; } catch (HelmRegistryException e) { log.error("failed to stop deployment ({})", versionKv(version), e); + stopDeploymentErrors.increment(); throw new RuntimeException(e); } }); } private void stopDeploymentOfRemovedVersion(ProjectVersion version) { + final Timer.Sample sample = Timer.start(); try { final WritableDeployment deployment = getOrCreateDeploymentForVersion(version); - HelmCommandUtils.uninstall(version); + helmCommands.uninstall(version); deploymentRepository.deleteById(deployment.getId()); + sample.stop(stopDeploymentDurationTimer); } catch (HelmRegistryException e) { log.error("failed to stop deployment of removed version ({})", versionKv(version), e); + stopDeploymentErrors.increment(); throw new RuntimeException(e); } } diff --git a/src/main/java/io/oneko/kubernetes/impl/DeploymentStatusWatcher.java b/src/main/java/io/oneko/kubernetes/impl/DeploymentStatusWatcher.java index 10de065a..e235745e 100644 --- a/src/main/java/io/oneko/kubernetes/impl/DeploymentStatusWatcher.java +++ b/src/main/java/io/oneko/kubernetes/impl/DeploymentStatusWatcher.java @@ -1,10 +1,9 @@ package io.oneko.kubernetes.impl; -import static io.oneko.util.MoreStructuredArguments.projectKv; -import static io.oneko.util.MoreStructuredArguments.versionKv; -import static net.logstash.logback.argument.StructuredArguments.kv; -import static net.logstash.logback.argument.StructuredArguments.v; +import static io.oneko.util.MoreStructuredArguments.*; +import static net.logstash.logback.argument.StructuredArguments.*; +import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.stream.Collectors; @@ -12,11 +11,13 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import io.oneko.event.CurrentEventTrigger; import io.oneko.event.EventTrigger; import io.oneko.event.ScheduledTask; +import io.oneko.helm.HelmCommands; import io.oneko.helm.HelmRegistryException; -import io.oneko.helm.util.HelmCommandUtils; import io.oneko.helmapi.model.Status; import io.oneko.kubernetes.deployments.DeployableStatus; import io.oneko.kubernetes.deployments.Deployment; @@ -24,6 +25,7 @@ import io.oneko.kubernetes.deployments.DesiredState; import io.oneko.kubernetes.deployments.ReadableDeployment; import io.oneko.kubernetes.deployments.WritableDeployment; +import io.oneko.metrics.MetricNameBuilder; import io.oneko.project.ProjectRepository; import io.oneko.project.ProjectVersion; import io.oneko.project.ProjectVersionLock; @@ -31,12 +33,10 @@ import io.oneko.project.WritableProjectVersion; import io.oneko.websocket.SessionWebSocketHandler; import io.oneko.websocket.message.DeploymentStatusChangedMessage; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; @Component @Slf4j -@AllArgsConstructor class DeploymentStatusWatcher { private final ProjectRepository projectRepository; @@ -44,24 +44,51 @@ class DeploymentStatusWatcher { private final SessionWebSocketHandler webSocketHandler; private final HelmStatusToDeploymentMapper helmStatusToDeploymentMapper; private final CurrentEventTrigger currentEventTrigger; - + private final HelmCommands helmCommands; private final ProjectVersionLock projectVersionLock; + private final Timer checkDeploymentStatusTimer; + + DeploymentStatusWatcher(ProjectRepository projectRepository, + DeploymentRepository deploymentRepository, + SessionWebSocketHandler webSocketHandler, + HelmStatusToDeploymentMapper helmStatusToDeploymentMapper, + CurrentEventTrigger currentEventTrigger, + HelmCommands helmCommands, + ProjectVersionLock projectVersionLock, + MeterRegistry meterRegistry) { + this.projectRepository = projectRepository; + this.deploymentRepository = deploymentRepository; + this.webSocketHandler = webSocketHandler; + this.helmStatusToDeploymentMapper = helmStatusToDeploymentMapper; + this.currentEventTrigger = currentEventTrigger; + this.helmCommands = helmCommands; + this.projectVersionLock = projectVersionLock; + this.checkDeploymentStatusTimer = Timer.builder(new MetricNameBuilder().durationOf("kubernetes.deployment.status.check").build()) + .description("the time it takes to iterate over all relevant deployments and update their status") + .publishPercentileHistogram() + .minimumExpectedValue(Duration.ofMillis(500)) + .register(meterRegistry); + } + + private EventTrigger asTrigger() { return new ScheduledTask("Kubernetes Deployment Status Watcher"); } @Scheduled(fixedRate = 10_000) protected void updateProjectStatus() { - final List writableVersions = projectRepository.getAll().stream() - .map(ReadableProject::writable) - .flatMap(writableProject -> writableProject.getVersions().stream()) - .filter(this::shouldScanDeployable) - .collect(Collectors.toList()); - - try (var ignored = currentEventTrigger.forTryBlock(asTrigger())) { - writableVersions.forEach(this::scanResourcesForDeployable); - } + checkDeploymentStatusTimer.record(() -> { + final List writableVersions = projectRepository.getAll().stream() + .map(ReadableProject::writable) + .flatMap(writableProject -> writableProject.getVersions().stream()) + .filter(this::shouldScanDeployable) + .collect(Collectors.toList()); + + try (var ignored = currentEventTrigger.forTryBlock(asTrigger())) { + writableVersions.forEach(this::scanResourcesForDeployable); + } + }); } private boolean shouldScanDeployable(ProjectVersion projectVersion) { @@ -72,7 +99,7 @@ private boolean shouldScanDeployable(ProjectVersion projectVersion) { private void scanResourcesForDeployable(ProjectVersion projectVersion) { projectVersionLock.doWithProjectVersionLock(projectVersion, () -> { try { - final List statuses = HelmCommandUtils.status(projectVersion); + final List statuses = helmCommands.status(projectVersion); if (statuses.isEmpty()) { cleanUpOnDeploymentRemoved(projectVersion); diff --git a/src/main/java/io/oneko/kubernetes/impl/KubernetesAccess.java b/src/main/java/io/oneko/kubernetes/impl/KubernetesAccess.java index 34068471..ea1ba2ca 100644 --- a/src/main/java/io/oneko/kubernetes/impl/KubernetesAccess.java +++ b/src/main/java/io/oneko/kubernetes/impl/KubernetesAccess.java @@ -2,9 +2,6 @@ import static net.logstash.logback.argument.StructuredArguments.*; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Base64; import java.util.HashMap; @@ -18,11 +15,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.LocalObjectReference; import io.fabric8.kubernetes.api.model.Namespace; import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.api.model.SecretBuilder; import io.fabric8.kubernetes.api.model.ServiceAccount; @@ -30,8 +25,11 @@ import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.KubernetesClient; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import io.oneko.event.EventDispatcher; import io.oneko.kubernetes.NamespaceCreatedEvent; +import io.oneko.metrics.MetricNameBuilder; import lombok.extern.slf4j.Slf4j; /** @@ -46,9 +44,16 @@ public class KubernetesAccess { private final KubernetesClient kubernetesClient; private final EventDispatcher eventDispatcher; + private final Timer createNamespaceTimer; + private final Timer deleteNamespaceTimer; + private final Timer createOrUpdateImagePullSecretTimer; + private final Timer deleteImagePullSecretTimer; + private final Timer patchServiceAccountTimer; + public KubernetesAccess(@Value("${kubernetes.server.url:}") final String masterUrl, @Value("${kubernetes.auth.token:}") final String token, - EventDispatcher eventDispatcher) { + EventDispatcher eventDispatcher, + MeterRegistry meterRegistry) { this.eventDispatcher = eventDispatcher; ConfigBuilder configBuilder = new ConfigBuilder(); @@ -66,17 +71,24 @@ public KubernetesAccess(@Value("${kubernetes.server.url:}") final String masterU .build(); kubernetesClient = new DefaultKubernetesClient(config); + + createNamespaceTimer = timer("namespace", "create", meterRegistry); + deleteNamespaceTimer = timer("namespace", "delete", meterRegistry); + createOrUpdateImagePullSecretTimer = timer("imagePullSecret", "createOrUpdate", meterRegistry); + deleteImagePullSecretTimer = timer("imagePullSecret", "delete", meterRegistry); + patchServiceAccountTimer = timer("serviceAccount", "patch", meterRegistry); } - List getPodsByLabelInNameSpace(String nameSpace, Map.Entry label) { - return kubernetesClient.pods() - .inNamespace(nameSpace) - .withLabel(label.getKey(), label.getValue()) - .list() - .getItems(); + private Timer timer(String resource, String action, MeterRegistry meterRegistry) { + return Timer.builder(new MetricNameBuilder().durationOf("kubernetes.api.request").build()) + .publishPercentileHistogram() + .tag("resource", resource) + .tag("action", action) + .register(meterRegistry); } Namespace createNamespaceIfNotExistent(String namespace) { + final Timer.Sample sample = Timer.start(); Namespace existingNamespace = kubernetesClient.namespaces().withName(namespace).get(); if (existingNamespace != null) { return existingNamespace; @@ -90,11 +102,12 @@ Namespace createNamespaceIfNotExistent(String namespace) { kubernetesClient.namespaces().create(newNameSpace); eventDispatcher.dispatch(new NamespaceCreatedEvent(newNameSpace.getMetadata().getName())); - + sample.stop(createNamespaceTimer); return newNameSpace; } Secret createOrUpdateImagePullSecretInNamespace(String namespace, String secretName, String userName, String password, String url) throws JsonProcessingException { + final Timer.Sample sample = Timer.start(); final Secret existingSecret = kubernetesClient.secrets() .inNamespace(namespace) .withName(secretName).get(); @@ -116,13 +129,15 @@ Secret createOrUpdateImagePullSecretInNamespace(String namespace, String secretN final HashMap dataMap = new HashMap<>(); dataMap.put(".dockerconfigjson", new String(Base64.getEncoder().encode(dockerConfigJson.getBytes()))); - return kubernetesClient.secrets().inNamespace(namespace).createOrReplace(new SecretBuilder() + final Secret secret = kubernetesClient.secrets().inNamespace(namespace).createOrReplace(new SecretBuilder() .withApiVersion("v1") .withKind("Secret") .withData(dataMap) .withNewMetadata().withName(secretName).endMetadata() .withType("kubernetes.io/dockerconfigjson") .build()); + sample.stop(createOrUpdateImagePullSecretTimer); + return secret; } catch (JsonProcessingException e) { log.error("failed to create image pull secret due to a JsonProcessingException.", e); throw e; @@ -130,69 +145,69 @@ Secret createOrUpdateImagePullSecretInNamespace(String namespace, String secretN } void deleteImagePullSecretInNamespace(String namespace, String secretName) { - kubernetesClient.secrets() - .inNamespace(namespace) - .withName(secretName) - .delete(); + deleteImagePullSecretTimer.record(() -> { + kubernetesClient.secrets() + .inNamespace(namespace) + .withName(secretName) + .delete(); + }); } ServiceAccount addImagePullSecretToServiceAccountIfNecessary(String namespace, String imagePullSecretName) { - final ServiceAccount defaultServiceAccount = kubernetesClient.serviceAccounts() - .inNamespace(namespace) - .withName("default") - .get(); - - List imagePullSecrets = defaultServiceAccount.getImagePullSecrets(); - if (imagePullSecrets == null) { - imagePullSecrets = new ArrayList<>(); - } - - if (imagePullSecrets.stream().anyMatch(ips -> ips.getName().equals(imagePullSecretName))) { - return defaultServiceAccount; - } - - log.info("adding image pull secret to default service account ({}, {})", kv("image_pull_secret", imagePullSecretName), kv("namespace", namespace)); - imagePullSecrets.add(new LocalObjectReference(imagePullSecretName)); - defaultServiceAccount.setImagePullSecrets(imagePullSecrets); - return kubernetesClient.serviceAccounts() - .inNamespace(namespace) - .createOrReplace(defaultServiceAccount); + return patchServiceAccountTimer.record(() -> { + final ServiceAccount defaultServiceAccount = kubernetesClient.serviceAccounts() + .inNamespace(namespace) + .withName("default") + .get(); + + List imagePullSecrets = defaultServiceAccount.getImagePullSecrets(); + if (imagePullSecrets == null) { + imagePullSecrets = new ArrayList<>(); + } + + if (imagePullSecrets.stream().anyMatch(ips -> ips.getName().equals(imagePullSecretName))) { + return defaultServiceAccount; + } + + log.info("adding image pull secret to default service account ({}, {})", kv("image_pull_secret", imagePullSecretName), kv("namespace", namespace)); + imagePullSecrets.add(new LocalObjectReference(imagePullSecretName)); + defaultServiceAccount.setImagePullSecrets(imagePullSecrets); + return kubernetesClient.serviceAccounts() + .inNamespace(namespace) + .createOrReplace(defaultServiceAccount); + }); } ServiceAccount removeImagePullSecretFromServiceAccountIfNecessary(String namespace, String imagePullSecretName) { - final ServiceAccount defaultServiceAccount = kubernetesClient.serviceAccounts() - .inNamespace(namespace) - .withName("default") - .get(); - - List imagePullSecrets = defaultServiceAccount.getImagePullSecrets(); - if (imagePullSecrets == null) { - return defaultServiceAccount; - } - - if (imagePullSecrets.stream().noneMatch(ips -> ips.getName().equals(imagePullSecretName))) { - return defaultServiceAccount; - } - - imagePullSecrets.removeIf(ips -> ips.getName().equals(imagePullSecretName)); - - log.info("removed image pull secret from default service account ({}, {})", kv("image_pull_secret", imagePullSecretName), kv("namespace", namespace)); - imagePullSecrets.add(new LocalObjectReference(imagePullSecretName)); - defaultServiceAccount.setImagePullSecrets(imagePullSecrets); - return kubernetesClient.serviceAccounts() - .inNamespace(namespace) - .createOrReplace(defaultServiceAccount); + return patchServiceAccountTimer.record(() -> { + final ServiceAccount defaultServiceAccount = kubernetesClient.serviceAccounts() + .inNamespace(namespace) + .withName("default") + .get(); + + List imagePullSecrets = defaultServiceAccount.getImagePullSecrets(); + if (imagePullSecrets == null) { + return defaultServiceAccount; + } + + if (imagePullSecrets.stream().noneMatch(ips -> ips.getName().equals(imagePullSecretName))) { + return defaultServiceAccount; + } + + imagePullSecrets.removeIf(ips -> ips.getName().equals(imagePullSecretName)); + + log.info("removed image pull secret from default service account ({}, {})", kv("image_pull_secret", imagePullSecretName), kv("namespace", namespace)); + imagePullSecrets.add(new LocalObjectReference(imagePullSecretName)); + defaultServiceAccount.setImagePullSecrets(imagePullSecrets); + return kubernetesClient.serviceAccounts() + .inNamespace(namespace) + .createOrReplace(defaultServiceAccount); + }); } public void deleteNamespaceByName(String name) { - kubernetesClient.namespaces().withName(name).delete(); - } - - List loadResource(String staticContent) { - try (InputStream is = new ByteArrayInputStream(staticContent.getBytes())) { - return kubernetesClient.load(is).get(); - } catch (IOException e) { - throw new RuntimeException(e); - } + deleteNamespaceTimer.record(() -> { + kubernetesClient.namespaces().withName(name).delete(); + }); } } diff --git a/src/main/java/io/oneko/metrics/MetricNameBuilder.java b/src/main/java/io/oneko/metrics/MetricNameBuilder.java new file mode 100644 index 00000000..d9babb0c --- /dev/null +++ b/src/main/java/io/oneko/metrics/MetricNameBuilder.java @@ -0,0 +1,45 @@ +package io.oneko.metrics; + +/** + * Helper class for naming metrics in compliance with + * https://prometheus.io/docs/practices/naming/ + */ +public class MetricNameBuilder { + + public static final String SEPARATOR = "."; + public static final String DOMAIN = "oneko"; + + //units and similar, like size, total, seconds etc. + public static final String TOTAL = "total"; + public static final String DURATION = "duration"; + + private final StringBuilder delegate; + + public MetricNameBuilder() { + this.delegate = new StringBuilder(DOMAIN); + } + + public MetricNameBuilder amountOf(String sizedThing) { + delegate.append(SEPARATOR).append(sizedThing).append(SEPARATOR).append(TOTAL); + return this; + } + + public MetricNameBuilder durationOf(String measuredThing) { + delegate.append(SEPARATOR).append(measuredThing).append(SEPARATOR).append(DURATION); + return this; + } + + public MetricNameBuilder with(String part) { + delegate.append(SEPARATOR).append(part); + return this; + } + + public String build() { + return delegate.toString(); + } + + @Override + public String toString() { + return build(); + } +} diff --git a/src/main/java/io/oneko/websocket/SessionWebSocketHandler.java b/src/main/java/io/oneko/websocket/SessionWebSocketHandler.java index abc8112a..c9f0997f 100644 --- a/src/main/java/io/oneko/websocket/SessionWebSocketHandler.java +++ b/src/main/java/io/oneko/websocket/SessionWebSocketHandler.java @@ -17,6 +17,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.oneko.metrics.MetricNameBuilder; import io.oneko.websocket.message.ONekoWebSocketMessage; import lombok.extern.slf4j.Slf4j; @@ -28,8 +32,19 @@ public class SessionWebSocketHandler extends TextWebSocketHandler { private final Map sessionContextMap = new HashMap<>(); private final ObjectMapper objectMapper; - public SessionWebSocketHandler(ObjectMapper objectMapper) { + private final Counter receivedWebsocketMessageCounter; + private final Counter sentWebsocketMessageCounter; + + public SessionWebSocketHandler(ObjectMapper objectMapper, MeterRegistry meterRegistry) { this.objectMapper = objectMapper; + Gauge.builder(new MetricNameBuilder().amountOf("websocket.sessions").build(), sessionContextMap::size) + .register(meterRegistry); + this.receivedWebsocketMessageCounter = Counter.builder(new MetricNameBuilder().amountOf("websocket.messages").build()) + .tag("type", "received") + .register(meterRegistry); + this.sentWebsocketMessageCounter = Counter.builder(new MetricNameBuilder().amountOf("websocket.messages").build()) + .tag("type", "sent") + .register(meterRegistry); } @Override @@ -63,6 +78,7 @@ protected void handleTextMessage(WebSocketSession session, TextMessage message) } // Currently, we do not handle incoming webSocket messages, so we just log them log.trace("received websocket message ({})", kv("message", msgObj.toString())); + receivedWebsocketMessageCounter.increment(); } public void invalidateWsSession(String wsSessionId) { @@ -85,6 +101,7 @@ public void send(WebSocketSession session, ONekoWebSocketMessage message) { try { var textMessage = new TextMessage(Objects.requireNonNull(this.messageToPayload(message))); session.sendMessage(textMessage); + sentWebsocketMessageCounter.increment(); } catch (IOException e) { log.error("error while sending websocket message ({})", kv("message", message)); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index f1fee9ff..3b54cbdd 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -13,7 +13,6 @@ logging: path: logs level: io.oneko: debug - o-neko: activity: cleanup: @@ -33,6 +32,15 @@ springdoc: doc-expansion: none tags-sorter: alpha operations-sorter: alpha + +management: + endpoints: + web: + exposure: + include: health, info, prometheus + metrics: + tags: + application: ${spring.application.name} --- # Development profile spring: diff --git a/src/test/java/io/oneko/InMemoryTestBench.java b/src/test/java/io/oneko/InMemoryTestBench.java index 83fa5066..94b1d31a 100644 --- a/src/test/java/io/oneko/InMemoryTestBench.java +++ b/src/test/java/io/oneko/InMemoryTestBench.java @@ -1,5 +1,6 @@ package io.oneko; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.oneko.docker.DockerRegistryRepository; import io.oneko.docker.persistence.DockerRegistryInMemoryRepository; import io.oneko.event.CurrentEventTrigger; @@ -20,7 +21,7 @@ public class InMemoryTestBench { private InMemoryTestBench(){} public final CurrentEventTrigger currentEventTrigger = new CurrentEventTrigger(); - public final EventDispatcher eventDispatcher = new EventDispatcher(currentEventTrigger); + public final EventDispatcher eventDispatcher = new EventDispatcher(currentEventTrigger, new SimpleMeterRegistry()); public final DockerRegistryRepository dockerRegistryRepository = new DockerRegistryInMemoryRepository(eventDispatcher); public final ProjectRepository projectRepository = new ProjectInMemoryRepository(eventDispatcher); diff --git a/src/test/java/io/oneko/docker/persistence/DockerRegistryInMemoryRepositoryTest.java b/src/test/java/io/oneko/docker/persistence/DockerRegistryInMemoryRepositoryTest.java index 551cf3bf..2593f8fc 100644 --- a/src/test/java/io/oneko/docker/persistence/DockerRegistryInMemoryRepositoryTest.java +++ b/src/test/java/io/oneko/docker/persistence/DockerRegistryInMemoryRepositoryTest.java @@ -2,6 +2,7 @@ import static org.assertj.core.api.Assertions.*; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.oneko.event.CurrentEventTrigger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -18,7 +19,7 @@ class DockerRegistryInMemoryRepositoryTest { @BeforeEach public void setup() { currentEventTrigger = new CurrentEventTrigger(); - dispatcher = new EventDispatcher(currentEventTrigger); + dispatcher = new EventDispatcher(currentEventTrigger, new SimpleMeterRegistry()); uut = new DockerRegistryInMemoryRepository(dispatcher); } diff --git a/src/test/java/io/oneko/event/EventDispatcherTest.java b/src/test/java/io/oneko/event/EventDispatcherTest.java index 1305e1ef..a954ad3e 100644 --- a/src/test/java/io/oneko/event/EventDispatcherTest.java +++ b/src/test/java/io/oneko/event/EventDispatcherTest.java @@ -9,11 +9,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + class EventDispatcherTest { private final List currentEvents = new ArrayList<>(); private final CurrentEventTrigger currentEventTrigger = new CurrentEventTrigger(); - private final EventDispatcher uut = new EventDispatcher(currentEventTrigger); + private final EventDispatcher uut = new EventDispatcher(currentEventTrigger, new SimpleMeterRegistry()); @BeforeEach diff --git a/src/test/java/io/oneko/helm/util/HelmCommandUtilsTest.java b/src/test/java/io/oneko/helm/HelmCommandsTest.java similarity index 63% rename from src/test/java/io/oneko/helm/util/HelmCommandUtilsTest.java rename to src/test/java/io/oneko/helm/HelmCommandsTest.java index e2ae46d6..2e7457b4 100644 --- a/src/test/java/io/oneko/helm/util/HelmCommandUtilsTest.java +++ b/src/test/java/io/oneko/helm/HelmCommandsTest.java @@ -1,4 +1,4 @@ -package io.oneko.helm.util; +package io.oneko.helm; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.*; @@ -7,46 +7,45 @@ import org.junit.jupiter.api.Test; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.oneko.project.Project; import io.oneko.project.ProjectVersion; -class HelmCommandUtilsTest { +class HelmCommandsTest { + + private HelmCommands uut = new HelmCommands(new SimpleMeterRegistry()); @Test void getLongReleaseNamePrefix() { - var projectUuid = UUID.fromString("5524b4a7-3b43-4879-8c0d-fe1c6a95c54f"); var versionUuid = UUID.fromString("39b30073-df7a-46d5-b85f-4d4377baa8c0"); var project = mock(Project.class); when(project.getName()).thenReturn("pnamelonglong"); - when(project.getId()).thenReturn(projectUuid); var version = mock(ProjectVersion.class); - when(version.getName()).thenReturn("vnamelonglong"); + when(version.getName()).thenReturn("vnamelonglonglonglong"); when(version.getId()).thenReturn(versionUuid); when(version.getProject()).thenReturn(project); - var prefix = HelmCommandUtils.getReleaseNamePrefix(version); + var prefix = uut.getReleaseNamePrefix(version); assertThat(prefix).hasSizeLessThanOrEqualTo(38); - assertThat(prefix).isEqualTo("pnamelongl5524b4a7-vnamelongl39b30073"); + assertThat(prefix).isEqualTo("pnamelongl-vnamelonglonglongl-39b30073"); } @Test void getShortReleaseNamePrefix() { - var projectUuid = UUID.fromString("5524b4a7-3b43-4879-8c0d-fe1c6a95c54f"); var versionUuid = UUID.fromString("39b30073-df7a-46d5-b85f-4d4377baa8c0"); var project = mock(Project.class); when(project.getName()).thenReturn("pname"); - when(project.getId()).thenReturn(projectUuid); var version = mock(ProjectVersion.class); when(version.getName()).thenReturn("vname"); when(version.getId()).thenReturn(versionUuid); when(version.getProject()).thenReturn(project); - var prefix = HelmCommandUtils.getReleaseNamePrefix(version); + var prefix = uut.getReleaseNamePrefix(version); assertThat(prefix).hasSizeLessThanOrEqualTo(38); - assertThat(prefix).isEqualTo("pname5524b4a7-vname39b30073"); + assertThat(prefix).isEqualTo("pname-vname-39b30073"); } -} \ No newline at end of file +} diff --git a/src/test/java/io/oneko/project/persistence/EventAwareProjectRepositoryTest.java b/src/test/java/io/oneko/project/persistence/EventAwareProjectRepositoryTest.java index dfbd5711..fc89e174 100644 --- a/src/test/java/io/oneko/project/persistence/EventAwareProjectRepositoryTest.java +++ b/src/test/java/io/oneko/project/persistence/EventAwareProjectRepositoryTest.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.UUID; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.oneko.event.*; import io.oneko.project.event.EventAwareProjectRepository; import io.oneko.project.event.ProjectSavedEvent; @@ -25,7 +26,7 @@ class EventAwareProjectRepositoryTest { @BeforeEach void setup() { this.currentEvents = new ArrayList<>(); - EventDispatcher dispatcher = new EventDispatcher(currentEventTrigger); + EventDispatcher dispatcher = new EventDispatcher(currentEventTrigger, new SimpleMeterRegistry()); dispatcher.registerListener(this.currentEvents::add); this.uut = new ProjectInMemoryRepository(dispatcher); }