From 7aef558bb460a3137067b830f2ec36407599a731 Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Thu, 31 Oct 2024 17:02:12 -0300 Subject: [PATCH] chore: Add sidecar --- Cargo.lock | 390 ++++++++++++++++++++++++++++++++- Cargo.toml | 2 + bootstrap/stage1/crd.tf | 7 +- bootstrap/stage2/deployment.tf | 9 +- bootstrap/stage2/main.tf | 8 +- src/bin/operator.rs | 12 +- src/config.rs | 2 + src/controller.rs | 159 +++++++++++++- src/custom_resource.rs | 70 +++++- 9 files changed, 635 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 500060f..1c1ade3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,21 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.91" @@ -96,6 +111,12 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" @@ -161,6 +182,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "bytes" version = "1.8.0" @@ -185,8 +212,11 @@ version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ + "android-tzdata", + "iana-time-zone", "num-traits", "serde", + "windows-targets 0.52.6", ] [[package]] @@ -293,6 +323,8 @@ dependencies = [ "k8s-openapi", "kube", "lazy_static", + "prometheus-parse", + "reqwest", "schemars", "serde", "serde_json", @@ -328,6 +360,15 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "enum-ordinalize" version = "4.3.0" @@ -546,6 +587,25 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "h2" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -650,6 +710,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2", "http", "http-body", "httparse", @@ -712,6 +773,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.9" @@ -731,12 +808,45 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -756,12 +866,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ipnet" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "json-patch" version = "2.0.0" @@ -1252,6 +1386,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "811031bea65e5a401fb2e1f37d802cca6601e204ac463809a3189352d13b78a5" +dependencies = [ + "chrono", + "itertools", + "once_cell", + "regex", +] + [[package]] name = "quote" version = "1.0.36" @@ -1329,6 +1475,49 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "reqwest" +version = "0.12.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-registry", +] + [[package]] name = "ring" version = "0.17.8" @@ -1547,6 +1736,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -1660,6 +1861,36 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -1702,6 +1933,21 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.38.0" @@ -1776,7 +2022,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper", + "sync_wrapper 0.1.2", "tokio", "tokio-util", "tower-layer", @@ -1890,12 +2136,27 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicode-bidi" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -1908,6 +2169,17 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "valuable" version = "0.1.0" @@ -1941,6 +2213,83 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "web-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1963,6 +2312,45 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index b3cbbba..edeb206 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ tokio-native-tls = "0.3.1" tracing = "0.1.40" lazy_static = "1.5.0" tracing-subscriber = "0.3.18" +reqwest = "0.12.9" +prometheus-parse = "0.2.5" [profile.release] debug = true diff --git a/bootstrap/stage1/crd.tf b/bootstrap/stage1/crd.tf index aef8382..a2871e8 100644 --- a/bootstrap/stage1/crd.tf +++ b/bootstrap/stage1/crd.tf @@ -86,6 +86,10 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_do "seedInput" = { "type" = "string" } + "sidecarImage" = { + "nullable" = true + "type" = "string" + } } "required" = [ "commitInputs", @@ -109,8 +113,7 @@ resource "kubernetes_manifest" "customresourcedefinition_hydradoomnodes_hydra_do "type" = "string" } "transactions" = { - "format" = "uint" - "minimum" = 0 + "format" = "int64" "type" = "integer" } } diff --git a/bootstrap/stage2/deployment.tf b/bootstrap/stage2/deployment.tf index 63e7334..1411443 100644 --- a/bootstrap/stage2/deployment.tf +++ b/bootstrap/stage2/deployment.tf @@ -45,12 +45,17 @@ resource "kubernetes_deployment_v1" "operator" { env { name = "IMAGE" - value = var.hydra_pod_image + value = var.hydra_node_image } env { name = "OPEN_HEAD_IMAGE" - value = var.hydra_pod_open_head_image + value = var.open_head_image + } + + env { + name = "SIDECAR_IMAGE" + value = var.sidecar_image } env { diff --git a/bootstrap/stage2/main.tf b/bootstrap/stage2/main.tf index a528c6c..bdfe7cd 100644 --- a/bootstrap/stage2/main.tf +++ b/bootstrap/stage2/main.tf @@ -12,12 +12,16 @@ variable "image" { type = string } -variable "hydra_pod_image" { +variable "hydra_node_image" { type = string default = "ghcr.io/cardano-scaling/hydra-node" } -variable "hydra_pod_open_head_image" { +variable "open_head_image" { + type = string +} + +variable "sidecar_image" { type = string } diff --git a/src/bin/operator.rs b/src/bin/operator.rs index 4386e63..a268595 100644 --- a/src/bin/operator.rs +++ b/src/bin/operator.rs @@ -6,7 +6,7 @@ use tracing::{error, info, instrument}; use doom_patrol::{ config::Config, - controller::{error_policy, reconcile, K8sContext}, + controller::{error_policy, patch_statuses, reconcile, K8sContext}, custom_resource::HydraDoomNode, }; @@ -25,15 +25,17 @@ async fn main() -> Result<()> { // Create controller for MyApp custom resource let api: Api = Api::all(client); info!("Running controller."); - Controller::new(api, Default::default()) - .run(reconcile, error_policy, context) + let controller = Controller::new(api, Default::default()) + .run(reconcile, error_policy, context.clone()) .for_each(|res| async move { match res { Ok(o) => info!("Reconciled {:?}", o), Err(e) => error!("Reconcile failed: {:?}", e), } - }) - .await; + }); + let patch_statuses_controller = patch_statuses(context.clone()); + + let _ = tokio::join!(controller, patch_statuses_controller); Ok(()) } diff --git a/src/config.rs b/src/config.rs index e1f2c83..b28c66f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -13,6 +13,7 @@ pub fn get_config() -> &'static Config { pub struct Config { pub image: String, pub open_head_image: String, + pub sidecar_image: String, pub configmap: String, pub blockfrost_key: String, pub external_domain: String, @@ -24,6 +25,7 @@ impl Config { Self { image: env::var("IMAGE").unwrap_or("ghcr.io/cardano-scaling/hydra-node".into()), open_head_image: env::var("OPEN_HEAD_IMAGE").expect("Missing OPEN_HEAD_IMAGE env var"), + sidecar_image: env::var("SIDECAR_IMAGE").expect("Missing SIDECAR_IMAGE env var"), configmap: env::var("CONFIGMAP").expect("Missing CONFIGMAP env var"), blockfrost_key: env::var("BLOCKFROST_KEY").expect("Missing BLOCKFROST_KEY env var"), external_domain: env::var("EXTERNAL_DOMAIN").expect("Missing EXTERNAL_DOMAIN env var."), diff --git a/src/controller.rs b/src/controller.rs index 11133da..50c0616 100644 --- a/src/controller.rs +++ b/src/controller.rs @@ -1,19 +1,46 @@ use anyhow::bail; use k8s_openapi::api::{apps::v1::Deployment, core::v1::Service, networking::v1::Ingress}; use kube::{ - api::{DeleteParams, Patch, PatchParams}, + api::{DeleteParams, ListParams, Patch, PatchParams}, runtime::controller::Action, Api, Client, ResourceExt, }; use serde_json::json; use std::{collections::BTreeMap, sync::Arc, time::Duration}; use thiserror::Error; -use tracing::{error, info}; +use tracing::{error, info, warn}; use crate::{config::Config, custom_resource::HydraDoomNodeStatus}; use super::custom_resource::{HydraDoomNode, HYDRA_DOOM_NODE_FINALIZER}; +pub enum HydraDoomNodeState { + Offline, + Online, + HeadIsInitializing, + HeadIsOpen, +} +impl From for HydraDoomNodeState { + fn from(value: f64) -> Self { + match value { + 1.0 => Self::Online, + 2.0 => Self::HeadIsInitializing, + 3.0 => Self::HeadIsOpen, + _ => Self::Offline, + } + } +} +impl From for String { + fn from(val: HydraDoomNodeState) -> Self { + match val { + HydraDoomNodeState::Offline => "Offline".to_string(), + HydraDoomNodeState::Online => "Online".to_string(), + HydraDoomNodeState::HeadIsInitializing => "HeadIsInitializing".to_string(), + HydraDoomNodeState::HeadIsOpen => "HeadIsOpen".to_string(), + } + } +} + pub struct K8sConstants { pub config_dir: String, pub data_dir: String, @@ -22,6 +49,10 @@ pub struct K8sConstants { pub port: i32, pub ingress_class_name: String, pub ingress_annotations: BTreeMap, + pub metrics_port: i32, + pub metrics_endpoint: String, + pub state_metric: String, + pub transactions_metric: String, } impl Default for K8sConstants { fn default() -> Self { @@ -31,6 +62,10 @@ impl Default for K8sConstants { persistence_dir: "/var/persistence".to_string(), node_port: 5001, port: 4001, + metrics_port: 8000, + metrics_endpoint: "/metrics".to_string(), + state_metric: "hydra_doom_node_state".to_string(), + transactions_metric: "hydra_doom_node_transactions".to_string(), ingress_class_name: "nginx".to_string(), ingress_annotations: [ ( @@ -247,6 +282,126 @@ impl K8sContext { Err(e) => Err(e.into()), } } + + async fn get_status_from_crd(&self, crd: &HydraDoomNode) -> HydraDoomNodeStatus { + let url = format!( + "http://{}:{}{}", + crd.internal_host(), + self.constants.metrics_port, + self.constants.metrics_endpoint + ); + let default = HydraDoomNodeStatus::offline(crd, &self.config, &self.constants); + + match reqwest::get(&url).await { + Ok(response) => match response.text().await { + Ok(body) => { + let lines: Vec<_> = body.lines().map(|s| Ok(s.to_owned())).collect(); + match prometheus_parse::Scrape::parse(lines.into_iter()) { + Ok(metrics) => { + let state = metrics + .clone() + .samples + .into_iter() + .find(|sample| sample.metric == self.constants.state_metric) + .map(|sample| match sample.value { + prometheus_parse::Value::Gauge(value) => { + HydraDoomNodeState::from(value) + } + _ => HydraDoomNodeState::Offline, + }); + + let transactions = metrics + .clone() + .samples + .into_iter() + .find(|sample| sample.metric == self.constants.state_metric) + .map(|sample| match sample.value { + prometheus_parse::Value::Counter(count) => count.round() as i64, + _ => 0, + }); + match (state, transactions) { + (Some(state), Some(transactions)) => HydraDoomNodeStatus { + transactions, + state: state.into(), + local_url: format!( + "ws://{}:{}", + crd.internal_host(), + self.constants.port + ), + external_url: format!( + "ws://{}:{}", + crd.external_host(&self.config, &self.constants), + self.config.external_port + ), + }, + _ => default, + } + } + Err(err) => { + warn!( + err = err.to_string(), + "Failed to parse metrics for {}", + crd.name_any() + ); + default + } + } + } + Err(err) => { + warn!( + err = err.to_string(), + "Failed to parse request response to metrics endpoint for {}", + crd.name_any() + ); + default + } + }, + Err(err) => { + warn!( + err = err.to_string(), + "Failed to request metrics for {}", + crd.name_any() + ); + default + } + } + } + + async fn patch_statuses(&self) -> anyhow::Result<()> { + let api: Api = Api::all(self.client.clone()); + + let crds = api.list(&ListParams::default()).await?; + + let mut awaitables = vec![]; + for crd in &crds { + awaitables.push(async { + let name = crd.name_any(); + if let Err(err) = api + .patch_status( + &name, + &PatchParams::default(), + &Patch::Merge(json!({ "status": self.get_status_from_crd(crd).await })), + ) + .await + { + warn!(err = err.to_string(), "Failed to status for CRD."); + }; + }) + } + + futures::future::join_all(awaitables).await; + + Ok(()) + } +} + +pub async fn patch_statuses(context: Arc) -> Result<()> { + info!("Running status patcher loop."); + + loop { + context.patch_statuses().await?; + tokio::time::sleep(Duration::from_secs(5)).await; + } } // Auxiliary error value because K8s controller api doesnt go along with anyhow. diff --git a/src/custom_resource.rs b/src/custom_resource.rs index acc6157..a3c216f 100644 --- a/src/custom_resource.rs +++ b/src/custom_resource.rs @@ -41,6 +41,7 @@ pub static HYDRA_DOOM_NODE_FINALIZER: &str = "hydradoomnode/finalizer"; pub struct HydraDoomNodeSpec { pub image: Option, pub open_head_image: Option, + pub sidecar_image: Option, pub configmap: Option, pub network_id: u8, pub seed_input: String, @@ -56,7 +57,21 @@ pub struct HydraDoomNodeStatus { pub local_url: String, pub external_url: String, pub state: String, - pub transactions: usize, + pub transactions: i64, +} +impl HydraDoomNodeStatus { + pub fn offline(crd: &HydraDoomNode, config: &Config, constants: &K8sConstants) -> Self { + Self { + state: "Offline".to_string(), + transactions: 0, + local_url: format!("ws://{}:{}", crd.internal_host(), constants.port), + external_url: format!( + "ws://{}:{}", + crd.external_host(config, constants), + config.external_port + ), + } + } } impl HydraDoomNode { @@ -185,6 +200,29 @@ impl HydraDoomNode { resources: None, // TODO: This should be parameterizable ..Default::default() }, + Container { + name: "sidecar".to_string(), + image: Some( + self.spec + .sidecar_image + .clone() + .unwrap_or(config.sidecar_image.clone()), + ), + args: Some(vec![ + "--host".to_string(), + "localhost".to_string(), + "--port".to_string(), + "0.0.0.0".to_string(), + constants.port.to_string(), + ]), + ports: Some(vec![ContainerPort { + name: Some("metrics".to_string()), + container_port: constants.metrics_port, + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + ..Default::default() + }, // Container { // name: "open-head".to_string(), // image: Some(self.spec.open_head_image.clone()), @@ -236,16 +274,28 @@ impl HydraDoomNode { }, spec: Some(ServiceSpec { selector: Some(labels), - ports: Some(vec![ServicePort { - port: constants.port, - target_port: Some( - k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int( - constants.port, + ports: Some(vec![ + ServicePort { + port: constants.port, + target_port: Some( + k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int( + constants.port, + ), ), - ), - protocol: Some("TCP".to_string()), - ..Default::default() - }]), + protocol: Some("TCP".to_string()), + ..Default::default() + }, + ServicePort { + port: constants.metrics_port, + target_port: Some( + k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int( + constants.metrics_port, + ), + ), + protocol: Some("TCP".to_string()), + ..Default::default() + }, + ]), type_: Some("ClusterIP".to_string()), ..Default::default() }),