diff --git a/.gitignore b/.gitignore index 6985cf1b..5ea5efc1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ Cargo.lock # MSVC Windows builds of rustc generate these, which store debugging information *.pdb + +.vscode/ diff --git a/Cargo.lock b/Cargo.lock index 04d142a7..b78638fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -600,7 +600,7 @@ checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -678,6 +678,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "bacalhau" +version = "0.1.0" +dependencies = [ + "handlebars", + "reqwest", + "section", + "serde", + "serde_json", + "serde_yaml", + "stub", + "tokio", + "tokio-stream", + "tokio-util", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -767,7 +783,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "syn_derive", ] @@ -927,7 +943,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1397,7 +1413,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1508,6 +1524,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "handlebars" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faa67bab9ff362228eb3d00bd024a4965d8231bbb7921167f0cfa66c6626b225" +dependencies = [ + "log", + "pest", + "pest_derive", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2135,6 +2165,7 @@ name = "myceliald" version = "0.2.0" dependencies = [ "anyhow", + "bacalhau", "base64 0.21.5", "clap", "common", @@ -2406,7 +2437,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -2547,6 +2578,51 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcd6ab1236bbdb3a49027e920e693192ebfe8913f6d60e294de57463a493cfde" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a31940305ffc96863a735bef7c7994a00b325a7138fdbc5bda0f1a0476d3275" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "pest_meta" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ff62f5259e53b78d1af898941cdcdccfae7385cf7d793a6e55de5d05bb4b7d" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -2564,7 +2640,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -2684,9 +2760,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -2733,9 +2809,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -3077,7 +3153,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.42", + "syn 2.0.48", "walkdir", ] @@ -3266,22 +3342,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3326,6 +3402,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" +dependencies = [ + "indexmap 2.1.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "server" version = "0.2.0" @@ -3815,7 +3904,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3845,9 +3934,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.42" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d0a2c048d661a1a59fcd7355baa232f7ed34e0ee4df2eef3c1c1c0d3852d8" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -3863,7 +3952,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3947,7 +4036,7 @@ checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -4041,7 +4130,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -4207,7 +4296,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -4241,6 +4330,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "unicase" version = "2.7.0" @@ -4289,6 +4384,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unsafe-libyaml" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" + [[package]] name = "untrusted" version = "0.7.1" @@ -4343,7 +4444,7 @@ checksum = "f49e7f3f3db8040a100710a11932239fd30697115e2ba4107080d8252939845e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -4404,7 +4505,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -4438,7 +4539,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4716,7 +4817,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5e3a409f..d60075d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "pipe/runtime", "pipe/arrow_msg", "pipe/section", + "pipe/section/section_impls/bacalhau", "pipe/section/section_impls/hello_world", "pipe/section/section_impls/sqlite_connector", "pipe/section/section_impls/excel_connector", diff --git a/common/src/lib.rs b/common/src/lib.rs index 9048d7c1..1aee2c52 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -53,6 +53,7 @@ pub enum Destination { Sqlite_Connector(SqliteConnectorConfig), Snowflake(SnowflakeConfig), Sqlite_Physical_Replication(SqlitePhysicalReplicationDestinationConfig), + Bacalhau(BacalhauDestinationConfig), Hello_World(HelloWorldDestinationConfig), Kafka(KafkaDestinationConfig), Postgres_Connector(PostgresConnectorDestinationConfig), @@ -115,6 +116,14 @@ pub struct SqlitePhysicalReplicationSourceConfig { // database path } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BacalhauDestinationConfig { + #[serde(flatten)] + pub common_attrs: CommonAttrs, + pub job: String, + pub jobstore: String, +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct HelloWorldSourceConfig { #[serde(flatten)] diff --git a/console/components/Flow/index.tsx b/console/components/Flow/index.tsx index 8cfa1690..3324b277 100644 --- a/console/components/Flow/index.tsx +++ b/console/components/Flow/index.tsx @@ -40,6 +40,7 @@ import { KafkaSourceNode, SnowflakeSourceNode, SnowflakeDestinationNode, + BacalhauDestinationNode, HelloWorldSourceNode, HelloWorldDestinationNode, ExcelSourceNode, @@ -73,11 +74,10 @@ const useStyles = createStyles((theme) => ({ marginBottom: theme.spacing.md, "&:not(:last-of-type)": { - borderBottom: `${rem(1)} solid ${ - theme.colorScheme === "dark" - ? theme.colors.dark[4] - : theme.colors.gray[3] - }`, + borderBottom: `${rem(1)} solid ${theme.colorScheme === "dark" + ? theme.colors.dark[4] + : theme.colors.gray[3] + }`, }, }, @@ -184,6 +184,7 @@ const nodeTypes = { snowflake_destination: SnowflakeDestinationNode, hello_world_source: HelloWorldSourceNode, hello_world_destination: HelloWorldDestinationNode, + bacalhau_destination: BacalhauDestinationNode, postgres_connector_source: PostgresConnectorSourceNode, postgres_connector_destination: PostgresConnectorDestinationNode, mysql_connector_destination: MysqlConnectorDestinationNode, @@ -282,7 +283,7 @@ function NavbarSearch(props: NavbarSearchProps) { onDragStart={(event) => onDragStart(event, null, source, source)} draggable > - Mycelial Server + Mycelial Server , ]; }; @@ -341,7 +342,7 @@ async function getConfigs(token: string) { }); const result = await response.json(); return result; - } catch (error) {} + } catch (error) { } } const dagreGraph = new dagre.graphlib.Graph(); @@ -564,7 +565,7 @@ function Flow() { if (edge.data?.id) { let payload = { - configs: [{id: edge.data.id, pipe: pipe}] + configs: [{ id: edge.data.id, pipe: pipe }] } try { const response = await fetch("/api/pipe", { @@ -582,7 +583,7 @@ function Flow() { } } else { let payload = { - configs: [{pipe: pipe}] + configs: [{ pipe: pipe }] } const response = await fetch("/api/pipe", { method: "POST", @@ -605,7 +606,7 @@ function Flow() { id: id, }, } - } + } return ed; }); }) diff --git a/console/components/nodes.tsx b/console/components/nodes.tsx index 0dbd37cd..69d6ec4b 100644 --- a/console/components/nodes.tsx +++ b/console/components/nodes.tsx @@ -16,6 +16,107 @@ import styles from "@/components/Flow/Flow.module.css"; import { ClientContext } from "./context/clientContext"; import { ClientContextType } from "./@types/client"; + +const BacalhauDestinationNode: FC = memo(({ id, data, selected }) => { + const instance = useReactFlow(); + const { clients } = useContext(ClientContext) as ClientContextType; + + let initialValues = useMemo(() => { + return { + client: data.client ? data.client : "-", + job: data.job ? data.job : "job", + jobstore: data.jobstore ? data.jobstore : "jobstore", + }; + }, []); + + const handleChange = useCallback((name: string, value: string) => { + instance.setNodes((nodes) => + nodes.map((node) => { + if (node.id === id) { + node.data = { + ...node.data, + [name]: value, + }; + } + + return node; + }), + ); + }, []); + + useEffect(() => { + handleChange("client", initialValues.client); + handleChange("job", initialValues.job); + handleChange("jobstore", initialValues.jobstore); + }, []); + + let classNames = `${styles.customNode} `; + if (selected) { + classNames = classNames + `${styles.selected}`; + } + + const removeNode = useCallback((id: string) => { + const node = instance.getNode(id); + if (node === undefined) { + return; + } + instance.deleteElements({ + edges: getConnectedEdges([node], []), + nodes: [node], + }); + }, []); + + return ( +
+
+

HelloWorld Destination

+ +