From b38d95f6656693475c2b4f93b7d58630d828ad26 Mon Sep 17 00:00:00 2001 From: Ben Steer Date: Wed, 18 Sep 2024 17:32:33 +0100 Subject: [PATCH] Graphql logging (#1746) * initial logging * Fixed spans * Removed auth, tidied config * on way to fixing * Adding future test * fmt * Fixed for now * make the server actually shut down * this info isn't showing up for some reason * Pushed logger throughout * minor fixes * fix * merge * fmt --------- Co-authored-by: Ben Steer Co-authored-by: Ben Steer Co-authored-by: Lucas Jeub --- Cargo.lock | 508 ++++++++++++------ Cargo.toml | 9 +- examples/rust/Cargo.toml | 1 + examples/rust/src/bin/bench/main.rs | 17 +- examples/rust/src/bin/btc/main.rs | 10 +- examples/rust/src/bin/crypto/main.rs | 15 +- examples/rust/src/bin/hulongbay/main.rs | 37 +- examples/rust/src/bin/lotr/main.rs | 9 +- examples/rust/src/bin/pokec/main.rs | 10 +- python/python/raphtory/__init__.pyi | 24 +- python/python/raphtory/graphql/__init__.pyi | 16 +- python/python/raphtory/vectors/__init__.pyi | 144 +---- .../edit_graph/test_deprecated_save.py | 16 +- python/tests/graphql/misc/test_tracing.py | 38 ++ .../graphql/test_graph_file_time_stats.py | 24 +- .../test_loaders/test_load_from_pandas.py | 236 ++++---- .../test_loaders/test_load_from_parquet.py | 113 ++-- python/tests/test_vectors.py | 16 +- raphtory-api/Cargo.toml | 3 +- raphtory-api/src/core/utils/logging.rs | 41 ++ raphtory-api/src/core/utils/mod.rs | 1 + raphtory-benchmark/Cargo.toml | 1 + raphtory-benchmark/bin/main.rs | 42 +- raphtory-benchmark/src/common/mod.rs | 5 +- raphtory-cypher/Cargo.toml | 1 + raphtory-cypher/examples/raphtory_cypher.rs | 12 +- .../src/executor/table_provider/edge.rs | 5 +- raphtory-cypher/src/hop/execution.rs | 2 +- raphtory-cypher/src/hop/rule.rs | 18 +- raphtory-cypher/src/parser/mod.rs | 7 +- raphtory-graphql/Cargo.toml | 3 +- raphtory-graphql/src/azure_auth/common.rs | 245 --------- raphtory-graphql/src/azure_auth/mod.rs | 3 - .../src/azure_auth/token_middleware.rs | 104 ---- raphtory-graphql/src/config/app_config.rs | 144 +++++ raphtory-graphql/src/config/cache_config.rs | 15 + raphtory-graphql/src/config/log_config.rs | 21 + raphtory-graphql/src/config/mod.rs | 51 ++ raphtory-graphql/src/config/otlp_config.rs | 71 +++ raphtory-graphql/src/data.rs | 21 +- raphtory-graphql/src/lib.rs | 7 +- raphtory-graphql/src/main.rs | 6 +- .../src/model/algorithms/global_search.rs | 3 +- .../src/model/algorithms/similarity_search.rs | 3 +- raphtory-graphql/src/model/graph/graph.rs | 15 +- raphtory-graphql/src/model/graph/graphs.rs | 1 + .../src/model/graph/mutable_graph.rs | 4 + raphtory-graphql/src/model/graph/node.rs | 6 + raphtory-graphql/src/model/graph/property.rs | 18 +- raphtory-graphql/src/observability/mod.rs | 2 +- .../src/observability/open_telemetry.rs | 203 +++++++ raphtory-graphql/src/observability/tracing.rs | 50 -- raphtory-graphql/src/python/client/mod.rs | 4 +- .../src/python/client/raphtory_client.rs | 3 +- .../src/python/server/running_server.rs | 13 +- raphtory-graphql/src/python/server/server.rs | 23 +- raphtory-graphql/src/server.rs | 222 +++----- raphtory-graphql/src/server_config.rs | 226 -------- raphtory/Cargo.toml | 2 +- raphtory/src/algorithms/algorithm_result.rs | 6 +- .../algorithms/community_detection/louvain.rs | 6 +- .../community_detection/modularity.rs | 8 +- raphtory/src/algorithms/components/lcc.rs | 3 +- .../algorithms/dynamics/temporal/epidemics.rs | 7 +- .../local_temporal_three_node_motifs.rs | 27 +- .../algorithms/motifs/three_node_motifs.rs | 8 +- .../pathing/single_source_shortest_path.rs | 5 +- .../pathing/temporal_reachability.rs | 4 - .../core/entities/nodes/structure/adjset.rs | 7 +- raphtory/src/db/graph/graph.rs | 41 +- raphtory/src/db/graph/views/deletion_graph.rs | 13 +- raphtory/src/db/graph/views/window_graph.rs | 22 +- raphtory/src/disk_graph/mod.rs | 3 +- raphtory/src/graph_loader/company_house.rs | 10 +- raphtory/src/graph_loader/karate_club.rs | 7 +- raphtory/src/graph_loader/lotr_graph.rs | 5 +- raphtory/src/graph_loader/mod.rs | 5 +- .../src/graph_loader/reddit_hyperlinks.rs | 7 +- raphtory/src/graph_loader/stable_coins.rs | 7 +- .../src/graphgen/preferential_attachment.rs | 10 +- raphtory/src/graphgen/random_attachment.rs | 7 +- raphtory/src/io/csv_loader.rs | 30 +- raphtory/src/io/json_loader.rs | 5 +- raphtory/src/lib.rs | 2 + .../src/python/graph/io/pandas_loaders.rs | 5 +- raphtory/src/python/graph/properties/props.rs | 1 - raphtory/src/search/mod.rs | 10 +- raphtory/src/serialise/incremental.rs | 6 +- raphtory/src/serialise/serialise.rs | 13 +- raphtory/src/vectors/embeddings.rs | 7 +- .../src/vectors/similarity_search_utils.rs | 1 - raphtory/src/vectors/template.rs | 7 +- raphtory/src/vectors/vectorisable.rs | 7 +- 93 files changed, 1706 insertions(+), 1476 deletions(-) create mode 100644 python/tests/graphql/misc/test_tracing.py create mode 100644 raphtory-api/src/core/utils/logging.rs delete mode 100644 raphtory-graphql/src/azure_auth/common.rs delete mode 100644 raphtory-graphql/src/azure_auth/mod.rs delete mode 100644 raphtory-graphql/src/azure_auth/token_middleware.rs create mode 100644 raphtory-graphql/src/config/app_config.rs create mode 100644 raphtory-graphql/src/config/cache_config.rs create mode 100644 raphtory-graphql/src/config/log_config.rs create mode 100644 raphtory-graphql/src/config/mod.rs create mode 100644 raphtory-graphql/src/config/otlp_config.rs create mode 100644 raphtory-graphql/src/observability/open_telemetry.rs delete mode 100644 raphtory-graphql/src/observability/tracing.rs delete mode 100644 raphtory-graphql/src/server_config.rs diff --git a/Cargo.lock b/Cargo.lock index 5c8f7eb012..278c25b366 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,7 +371,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap", + "indexmap 2.4.0", "lexical-core", "num", "serde", @@ -496,7 +496,7 @@ dependencies = [ "futures-util", "handlebars", "http 1.1.0", - "indexmap", + "indexmap 2.4.0", "mime", "multer", "num-traits", @@ -513,26 +513,26 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.9" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1141703c11c6ad4fa9b3b0e1e476dea01dbd18a44db00f949b804afaab2f344" +checksum = "72e2e26a6b44bc61df3ca8546402cf9204c28e30c06084cc8e75cd5e34d4f150" dependencies = [ "Inflector", "async-graphql-parser", "darling", - "proc-macro-crate 3.2.0", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", "strum", - "syn 2.0.77", + "syn 2.0.75", "thiserror", ] [[package]] name = "async-graphql-parser" -version = "7.0.9" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f66edcce4c38c18f7eb181fdf561c3d3aa2d644ce7358fc7a928c00a4ffef17" +checksum = "f801451484b4977d6fe67b29030f81353cabdcbb754e5a064f39493582dac0cf" dependencies = [ "async-graphql-value", "pest", @@ -559,12 +559,12 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.9" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0206011cad065420c27988f17dd7fe201a0e056b20c262209b7bffcd6fa176" +checksum = "69117c43c01d81a69890a9f5dd6235f2f027ca8d1ec62d6d3c5e01ca0edb4f2b" dependencies = [ "bytes", - "indexmap", + "indexmap 2.4.0", "serde", "serde_json", ] @@ -614,18 +614,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -649,12 +649,70 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.1", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backoff" version = "0.4.0" @@ -855,7 +913,7 @@ checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -902,9 +960,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.1.15" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" dependencies = [ "jobserver", "libc", @@ -925,9 +983,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" @@ -1056,7 +1114,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -1143,9 +1201,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "convert_case" @@ -1361,7 +1419,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -1372,7 +1430,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -1443,7 +1501,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.4.0", "itertools 0.12.1", "log", "num_cpus", @@ -1612,7 +1670,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.4.0", "itertools 0.12.1", "log", "paste", @@ -1641,7 +1699,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "hex", - "indexmap", + "indexmap 2.4.0", "itertools 0.12.1", "log", "paste", @@ -1687,7 +1745,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.4.0", "itertools 0.12.1", "log", "once_cell", @@ -1768,38 +1826,38 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] name = "derive_builder" -version = "0.20.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.20.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] name = "derive_builder_macro" -version = "0.20.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -1833,7 +1891,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -1891,7 +1949,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", "thiserror", ] @@ -1919,7 +1977,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -1928,8 +1986,11 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" dependencies = [ + "atty", + "humantime", "log", "regex", + "termcolor", ] [[package]] @@ -1974,6 +2035,7 @@ dependencies = [ "raphtory", "regex", "serde", + "tracing", ] [[package]] @@ -2005,9 +2067,9 @@ checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b" [[package]] name = "fastrand" -version = "2.1.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fixedbitset" @@ -2027,9 +2089,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" dependencies = [ "crc32fast", "miniz_oxide 0.8.0", @@ -2122,7 +2184,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -2224,7 +2286,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -2243,7 +2305,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -2275,6 +2337,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.13.2" @@ -2328,6 +2396,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.9" @@ -2519,7 +2596,20 @@ dependencies = [ "tokio", "tokio-rustls 0.26.0", "tower-service", - "webpki-roots 0.26.5", + "webpki-roots 0.26.3", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.4.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -2583,9 +2673,19 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2956,6 +3056,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -3072,14 +3178,13 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ - "hermit-abi 0.3.9", "libc", "wasi", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -3178,7 +3283,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.0", "url", - "webpki-roots 0.26.5", + "webpki-roots 0.26.3", ] [[package]] @@ -3188,14 +3293,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a0d57c55d2d1dc62a2b1d16a0a1079eb78d67c36bdf468d582ab4482ec7002" dependencies = [ "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] name = "nix" -version = "0.29.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -3335,9 +3440,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" dependencies = [ "memchr", ] @@ -3395,9 +3500,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "opentelemetry" -version = "0.23.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b69a91d4893e713e06f724597ad630f1fa76057a5e1026c0ca67054a9032a76" +checksum = "803801d3d3b71cd026851a53f974ea03df3d179cb758b260136a6c9e22e196af" dependencies = [ "futures-core", "futures-sink", @@ -3408,44 +3513,51 @@ dependencies = [ ] [[package]] -name = "opentelemetry-jaeger" -version = "0.22.0" +name = "opentelemetry-otlp" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501b471b67b746d9a07d4c29f8be00f952d1a2eca356922ede0098cbaddff19f" +checksum = "596b1719b3cab83addb20bcbffdf21575279d9436d9ccccfe651a3bf0ab5ab06" dependencies = [ "async-trait", "futures-core", - "futures-util", + "http 1.1.0", "opentelemetry", - "opentelemetry-semantic-conventions", + "opentelemetry-proto", "opentelemetry_sdk", - "thrift", + "prost", + "thiserror", "tokio", + "tonic", ] [[package]] -name = "opentelemetry-semantic-conventions" -version = "0.15.0" +name = "opentelemetry-proto" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1869fb4bb9b35c5ba8a1e40c9b128a7b4c010d07091e864a29da19e4fe2ca4d7" +checksum = "2c43620e8f93359eb7e627a3b16ee92d8585774986f24f2ab010817426c5ce61" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", +] [[package]] name = "opentelemetry_sdk" -version = "0.23.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae312d58eaa90a82d2e627fd86e075cf5230b3f11794e2ed74199ebbe572d4fd" +checksum = "e0da0d6b47a3dbc6e9c9e36a0520e25cf943e046843818faaa3f87365a548c82" dependencies = [ "async-trait", "futures-channel", "futures-executor", "futures-util", "glob", - "lazy_static", "once_cell", "opentelemetry", - "ordered-float 4.2.2", "percent-encoding", "rand", + "serde_json", "thiserror", "tokio", "tokio-stream", @@ -3501,7 +3613,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -3662,7 +3774,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -3683,7 +3795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.4.0", ] [[package]] @@ -3741,7 +3853,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -3801,9 +3913,9 @@ dependencies = [ [[package]] name = "poem" -version = "3.0.4" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1ba1c27f8f89e1bccdda0c680f72790545a11a8d8555819472f5839d7a8ca9d" +checksum = "14c2c9ce82b482f5d7b0db655b104fc80fc0d7de9fc099e9f4a40a96faaef6f1" dependencies = [ "base64 0.22.1", "bytes", @@ -3843,10 +3955,10 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a62fea1692d80a000126f9b28d865012a160b80000abb53ccf152b428222c155" dependencies = [ - "proc-macro-crate 3.2.0", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -3937,7 +4049,7 @@ dependencies = [ "ahash", "bytemuck", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.4.0", "num-traits", "once_cell", "polars-error", @@ -3997,12 +4109,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -4017,11 +4129,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.2.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "toml_edit 0.22.20", + "toml_edit 0.21.1", ] [[package]] @@ -4041,7 +4153,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", "version_check", "yansi 1.0.1", ] @@ -4068,9 +4180,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" dependencies = [ "bytes", "prost-derive", @@ -4078,9 +4190,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8650aabb6c35b860610e9cff5dc1af886c9e25073b7b1712a68972af4281302" +checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1" dependencies = [ "bytes", "heck 0.5.0", @@ -4093,37 +4205,37 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.77", + "syn 2.0.75", "tempfile", ] [[package]] name = "prost-derive" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" +checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" dependencies = [ "anyhow", "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] name = "prost-types" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519" +checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" dependencies = [ "prost", ] [[package]] name = "psm" -version = "0.1.22" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b1f9bf148c15500d44581654fb9260bc9d82970f3ef777a79a40534f6aa784f" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" dependencies = [ "cc", ] @@ -4177,7 +4289,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -4190,7 +4302,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -4244,9 +4356,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.4" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d2fb862b7ba45e615c1429def928f2e15f815bdf933b27a2d3824e224c1f46" +checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" dependencies = [ "bytes", "pin-project-lite", @@ -4262,9 +4374,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.7" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0a9b3a42929fad8a7c3de7f86ce0814cfa893328157672680e9fb1145549c5" +checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" dependencies = [ "bytes", "rand", @@ -4292,9 +4404,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -4364,6 +4476,7 @@ dependencies = [ "dotenv", "either", "enum_dispatch", + "env_logger", "flate2", "futures-util", "glam", @@ -4410,6 +4523,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tracing", "zip", ] @@ -4431,6 +4545,8 @@ dependencies = [ "rayon", "rustc-hash 2.0.0", "serde", + "tracing", + "tracing-subscriber", "twox-hash", ] @@ -4449,6 +4565,7 @@ dependencies = [ "rayon", "sorted_vector_map", "tempfile", + "tracing", ] [[package]] @@ -4479,6 +4596,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tracing", ] [[package]] @@ -4501,7 +4619,7 @@ dependencies = [ "oauth2", "once_cell", "opentelemetry", - "opentelemetry-jaeger", + "opentelemetry-otlp", "opentelemetry_sdk", "ordered-float 4.2.2", "parking_lot", @@ -4709,7 +4827,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.26.5", + "webpki-roots 0.26.3", "windows-registry", ] @@ -4811,18 +4929,18 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.6.0", "errno", @@ -4852,16 +4970,16 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.6", "subtle", "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.7.3" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" dependencies = [ "openssl-probe", "rustls-pemfile 2.1.3", @@ -4907,9 +5025,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ "ring", "rustls-pki-types", @@ -5027,9 +5145,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] @@ -5047,20 +5165,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ "itoa", "memchr", @@ -5300,7 +5418,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -5311,15 +5429,15 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.17" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "winapi", ] [[package]] @@ -5389,7 +5507,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -5411,9 +5529,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" dependencies = [ "proc-macro2", "quote", @@ -5628,6 +5746,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.3.0" @@ -5655,7 +5782,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -5668,15 +5795,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - [[package]] name = "thrift" version = "0.17.0" @@ -5685,9 +5803,7 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "log", "ordered-float 2.10.1", - "threadpool", ] [[package]] @@ -5757,31 +5873,32 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" dependencies = [ "backtrace", "bytes", "libc", "mio", + "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -5818,9 +5935,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.23.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" +checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" dependencies = [ "futures-util", "log", @@ -5869,7 +5986,18 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap", + "indexmap 2.4.0", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +dependencies = [ + "indexmap 2.4.0", "toml_datetime", "winnow 0.5.40", ] @@ -5880,13 +6008,43 @@ version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap", + "indexmap 2.4.0", "serde", "serde_spanned", "toml_datetime", "winnow 0.6.18", ] +[[package]] +name = "tonic" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f6ba989e4b2c58ae83d862d3a3e27690b6e3ae630d0deb59f3697f32aa88ad" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.6", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -5895,11 +6053,16 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -5933,7 +6096,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -5959,9 +6122,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.24.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f68803492bf28ab40aeccaecc7021096bd256baf7ca77c3d425d89b35a7be4e4" +checksum = "5eabc56d23707ad55ba2a0750fc24767125d5a0f51993ba41ad2c441cc7b8dea" dependencies = [ "js-sys", "once_cell", @@ -6007,9 +6170,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.23.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", "bytes", @@ -6020,6 +6183,7 @@ dependencies = [ "rand", "sha1", "thiserror", + "url", "utf-8", ] @@ -6233,7 +6397,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", "wasm-bindgen-shared", ] @@ -6267,7 +6431,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6301,7 +6465,7 @@ checksum = "4b8220be1fa9e4c889b30fd207d4906657e7e90b12e0e6b0c8b8d8709f5de021" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -6345,9 +6509,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.5" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" dependencies = [ "rustls-pki-types", ] @@ -6652,7 +6816,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -6672,7 +6836,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.75", ] [[package]] @@ -6691,7 +6855,7 @@ dependencies = [ "displaydoc", "flate2", "hmac", - "indexmap", + "indexmap 2.4.0", "lzma-rs", "memchr", "pbkdf2", diff --git a/Cargo.toml b/Cargo.toml index cf0c57f56a..3e4177e56a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,11 +111,11 @@ criterion = "0.5.1" crossbeam-channel = "0.5.11" base64 = "0.22.1" poem = { version = "3.0.1", features = ["cookie"] } -opentelemetry = "0.23.0" -opentelemetry_sdk = { version = "0.23.0", features = ["rt-tokio"] } -opentelemetry-jaeger = { version = "0.22.0", features = ["rt-tokio"] } +opentelemetry = "0.25.0" +opentelemetry_sdk = { version = "0.25.0", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.25.0"} tracing = "0.1.37" -tracing-opentelemetry = "0.24.0" +tracing-opentelemetry = "0.26.0" tracing-subscriber = { version = "0.3.16", features = ["std", "env-filter"] } indoc = "2.0.5" walkdir = "2" @@ -135,7 +135,6 @@ bytemuck = { version = "1.18.0", features = ["derive"] } ouroboros = "0.18.3" url = "2.2" base64-compat = { package = "base64-compat", version = "1.0.0" } - prost = "0.13.1" prost-types = "0.13.1" prost-build = "0.13.1" diff --git a/examples/rust/Cargo.toml b/examples/rust/Cargo.toml index a280794697..8721245bae 100644 --- a/examples/rust/Cargo.toml +++ b/examples/rust/Cargo.toml @@ -12,6 +12,7 @@ chrono = { workspace = true } regex = { workspace = true } serde = { workspace = true } itertools = { workspace = true } +tracing = { workspace = true } [[bin]] name = "btc" diff --git a/examples/rust/src/bin/bench/main.rs b/examples/rust/src/bin/bench/main.rs index d6cb8ee065..ad63c92313 100644 --- a/examples/rust/src/bin/bench/main.rs +++ b/examples/rust/src/bin/bench/main.rs @@ -1,5 +1,6 @@ use raphtory::{ - algorithms::centrality::pagerank::unweighted_page_rank, io::csv_loader::CsvLoader, prelude::*, + algorithms::centrality::pagerank::unweighted_page_rank, io::csv_loader::CsvLoader, + logging::global_info_logger, prelude::*, }; use serde::Deserialize; use std::{ @@ -7,6 +8,7 @@ use std::{ path::{Path, PathBuf}, time::Instant, }; +use tracing::info; #[derive(Deserialize, std::fmt::Debug)] pub struct Benchr { @@ -15,7 +17,8 @@ pub struct Benchr { } fn main() { - println!("Hello, world!"); + global_info_logger(); + info!("Hello, world!"); let args: Vec = env::args().collect(); let default_data_dir: PathBuf = [env!("CARGO_MANIFEST_DIR"), "src/bin/bench/data"] @@ -33,13 +36,13 @@ fn main() { } let encoded_data_dir = data_dir.join("graphdb.bincode"); - println!("Loading data"); + info!("Loading data"); let graph = if encoded_data_dir.exists() { let now = Instant::now(); let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); - println!( + info!( "Loaded graph from encoded data files {} with {} nodes, {} edges which took {} seconds", encoded_data_dir.to_str().unwrap(), g.count_nodes(), @@ -66,7 +69,7 @@ fn main() { }) .expect("Failed to load graph from CSV data files"); - println!( + info!( "Loaded graph from CSV data files {} with {} nodes, {} edges which took {} seconds", encoded_data_dir.to_str().unwrap(), g.count_nodes(), @@ -78,7 +81,7 @@ fn main() { g }; - println!("Data loaded\nPageRanking"); + info!("Data loaded\nPageRanking"); unweighted_page_rank(&graph, Some(25), Some(8), None, true, None); - println!("Done PR"); + info!("Done PR"); } diff --git a/examples/rust/src/bin/btc/main.rs b/examples/rust/src/bin/btc/main.rs index 1364085c42..0dc39c63a6 100644 --- a/examples/rust/src/bin/btc/main.rs +++ b/examples/rust/src/bin/btc/main.rs @@ -2,7 +2,7 @@ #![allow(dead_code)] use chrono::{DateTime, Utc}; -use raphtory::{io::csv_loader::CsvLoader, prelude::*}; +use raphtory::{io::csv_loader::CsvLoader, logging::global_info_logger, prelude::*}; use regex::Regex; use serde::Deserialize; use std::{ @@ -16,6 +16,7 @@ use std::{ thread::JoinHandle, time::Instant, }; +use tracing::info; #[derive(Deserialize, std::fmt::Debug)] pub struct Sent { @@ -38,6 +39,7 @@ pub struct Received { } fn main() { + global_info_logger(); let args: Vec = env::args().collect(); let default_data_dir: PathBuf = [env!("CARGO_MANIFEST_DIR"), "src/bin/btc/data"] @@ -65,7 +67,7 @@ fn main() { let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); - println!( + info!( "Loaded graph from path {} with {} nodes, {} edges, took {} seconds", encoded_data_dir.to_str().unwrap(), g.count_nodes(), @@ -87,7 +89,7 @@ fn main() { let time = sent.time.timestamp(); if src == test_v || dst == test_v { - println!("{} sent {} to {}", sent.addr, sent.amount_btc, sent.txn); + info!("{} sent {} to {}", sent.addr, sent.amount_btc, sent.txn); } g.add_edge( @@ -101,7 +103,7 @@ fn main() { }) .expect("Failed to load graph from CSV data files"); - println!( + info!( "Loaded graph from CSV data files {} with {} nodes, {} edges which took {} seconds", encoded_data_dir.to_str().unwrap(), g.count_nodes(), diff --git a/examples/rust/src/bin/crypto/main.rs b/examples/rust/src/bin/crypto/main.rs index 77d0a47091..61c59e418a 100644 --- a/examples/rust/src/bin/crypto/main.rs +++ b/examples/rust/src/bin/crypto/main.rs @@ -6,10 +6,13 @@ use raphtory::{ }, db::api::view::*, graph_loader::stable_coins::stable_coin_graph, + logging::global_info_logger, }; use std::{env, time::Instant}; +use tracing::info; fn main() { + global_info_logger(); let args: Vec = env::args().collect(); let data_dir = if args.len() < 2 { @@ -28,21 +31,21 @@ fn main() { vec!["Dai", "LUNC", "USD", "USDP", "USDT", "USTC"] ); - println!("Pagerank"); + info!("Pagerank"); let now = Instant::now(); let _ = unweighted_page_rank(&g, Some(20), None, None, true, None); - println!("Time taken: {} secs", now.elapsed().as_secs()); + info!("Time taken: {} secs", now.elapsed().as_secs()); let now = Instant::now(); let _ = unweighted_page_rank(&g, Some(20), None, None, true, None); - println!("Time taken: {} secs", now.elapsed().as_secs()); + info!("Time taken: {} secs", now.elapsed().as_secs()); let now = Instant::now(); let _ = unweighted_page_rank(&g.layers("USDT").unwrap(), Some(20), None, None, true, None); - println!("Time taken: {} secs", now.elapsed().as_secs()); + info!("Time taken: {} secs", now.elapsed().as_secs()); - println!("Generic taint"); + info!("Generic taint"); let now = Instant::now(); let _ = temporally_reachable_nodes( &g.layers("USDT").unwrap(), @@ -52,5 +55,5 @@ fn main() { vec!["0xd30b438df65f4f788563b2b3611bd6059bff4ad9"], None, ); - println!("Time taken: {} secs", now.elapsed().as_secs()); + info!("Time taken: {} secs", now.elapsed().as_secs()); } diff --git a/examples/rust/src/bin/hulongbay/main.rs b/examples/rust/src/bin/hulongbay/main.rs index 677bcb2073..263b1888b9 100644 --- a/examples/rust/src/bin/hulongbay/main.rs +++ b/examples/rust/src/bin/hulongbay/main.rs @@ -10,6 +10,7 @@ use raphtory::{ }, }, io::csv_loader::CsvLoader, + logging::global_info_logger, prelude::*, }; use regex::Regex; @@ -21,6 +22,7 @@ use std::{ path::Path, time::Instant, }; +use tracing::{error, info}; #[derive(Deserialize, Debug)] pub struct Edge { @@ -65,7 +67,7 @@ pub fn loader(data_dir: &Path) -> Result> { let now = Instant::now(); let g = Graph::decode(encoded_data_dir.as_path())?; - println!( + info!( "Loaded graph from path {} with {} nodes, {} edges, took {} seconds", encoded_data_dir.display(), g.count_nodes(), @@ -96,7 +98,7 @@ pub fn loader(data_dir: &Path) -> Result> { .unwrap(); })?; - println!( + info!( "Loaded graph from CSV data files {} with {} nodes, {} edges which took {} seconds", encoded_data_dir.display(), g.count_nodes(), @@ -110,6 +112,7 @@ pub fn loader(data_dir: &Path) -> Result> { } fn try_main() -> Result<(), Box> { + global_info_logger(); let args: Vec = env::args().collect(); let data_dir = Path::new(args.get(1).ok_or(MissingArgumentError)?); @@ -117,9 +120,9 @@ fn try_main() -> Result<(), Box> { let now = Instant::now(); let actual_tri_count = triangle_count(&graph, None); - println!("Actual triangle count: {:?}", actual_tri_count); + info!("Actual triangle count: {:?}", actual_tri_count); - println!( + info!( "Counting triangles took {} seconds", now.elapsed().as_secs() ); @@ -136,31 +139,31 @@ fn try_main() -> Result<(), Box> { .rev() .take(50) .for_each(|(cc, count)| { - println!("CC {} has {} nodes", cc, count); + info!("CC {} has {} nodes", cc, count); }); - println!( + info!( "Connected Components took {} seconds", now.elapsed().as_secs() ); let now = Instant::now(); let num_edges: usize = graph.nodes().out_degree().sum(); - println!( + info!( "Counting edges by summing degrees returned {} in {} seconds", num_edges, now.elapsed().as_secs() ); let earliest_time = graph.start().ok_or(GraphEmptyError)?; let latest_time = graph.end().ok_or(GraphEmptyError)?; - println!("graph time range: {}-{}", earliest_time, latest_time); + info!("graph time range: {}-{}", earliest_time, latest_time); let now = Instant::now(); let window = graph.window(i64::MIN, i64::MAX); - println!("Creating window took {} seconds", now.elapsed().as_secs()); + info!("Creating window took {} seconds", now.elapsed().as_secs()); let now = Instant::now(); let num_windowed_edges: usize = window.nodes().out_degree().sum(); - println!( + info!( "Counting edges in window by summing degrees returned {} in {} seconds", num_windowed_edges, now.elapsed().as_secs() @@ -168,7 +171,7 @@ fn try_main() -> Result<(), Box> { let now = Instant::now(); let num_windowed_edges2 = window.count_edges(); - println!( + info!( "Window num_edges returned {} in {} seconds", num_windowed_edges2, now.elapsed().as_secs() @@ -178,6 +181,7 @@ fn try_main() -> Result<(), Box> { } fn try_main_bm() -> Result<(), Box> { + global_info_logger(); let args: Vec = env::args().collect(); let data_dir = Path::new(args.get(1).ok_or(MissingArgumentError)?); @@ -185,18 +189,18 @@ fn try_main_bm() -> Result<(), Box> { let now = Instant::now(); let num_edges: usize = graph.nodes().iter().map(|v| v.out_degree()).sum(); - println!( + info!( "Counting edges by summing degrees returned {} in {} milliseconds", num_edges, now.elapsed().as_millis() ); let earliest_time = graph.start().ok_or(GraphEmptyError)?; let latest_time = graph.end().ok_or(GraphEmptyError)?; - println!("graph time range: {}-{}", earliest_time, latest_time); + info!("graph time range: {}-{}", earliest_time, latest_time); let now = Instant::now(); let num_edges2 = graph.count_edges(); - println!( + info!( "num_edges returned {} in {} milliseconds", num_edges2, now.elapsed().as_millis() @@ -204,7 +208,7 @@ fn try_main_bm() -> Result<(), Box> { let now = Instant::now(); let num_exploded_edges = graph.edges().explode().iter().count(); - println!( + info!( "counted {} exploded edges in {} milliseconds", num_exploded_edges, now.elapsed().as_millis() @@ -222,8 +226,9 @@ fn try_motif() -> Result<(), Box> { } fn main() { + global_info_logger(); if let Err(e) = try_motif() { - eprintln!("Failed: {}", e); + error!("Failed: {}", e); std::process::exit(1) } } diff --git a/examples/rust/src/bin/lotr/main.rs b/examples/rust/src/bin/lotr/main.rs index acaab631f3..d618e15d8a 100644 --- a/examples/rust/src/bin/lotr/main.rs +++ b/examples/rust/src/bin/lotr/main.rs @@ -1,6 +1,6 @@ use raphtory::{ algorithms::pathing::temporal_reachability::temporally_reachable_nodes, - io::csv_loader::CsvLoader, prelude::*, + io::csv_loader::CsvLoader, logging::global_info_logger, prelude::*, }; use serde::Deserialize; use std::{ @@ -8,6 +8,7 @@ use std::{ path::{Path, PathBuf}, time::Instant, }; +use tracing::info; #[derive(Deserialize, std::fmt::Debug)] pub struct Lotr { @@ -18,7 +19,7 @@ pub struct Lotr { fn main() { let args: Vec = env::args().collect(); - + global_info_logger(); let default_data_dir: PathBuf = [env!("CARGO_MANIFEST_DIR"), "src/bin/lotr/data"] .iter() .collect(); @@ -40,7 +41,7 @@ fn main() { let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); - println!( + info!( "Loaded graph from encoded data files {} with {} nodes, {} edges which took {} seconds", encoded_data_dir.to_str().unwrap(), g.count_nodes(), @@ -82,7 +83,7 @@ fn main() { }) .expect("Failed to load graph from CSV data files"); - println!( + info!( "Loaded graph from CSV data files {} with {} nodes, {} edges which took {} seconds", encoded_data_dir.to_str().unwrap(), g.count_nodes(), diff --git a/examples/rust/src/bin/pokec/main.rs b/examples/rust/src/bin/pokec/main.rs index 00edba5e11..b27bc464d2 100644 --- a/examples/rust/src/bin/pokec/main.rs +++ b/examples/rust/src/bin/pokec/main.rs @@ -4,10 +4,12 @@ use raphtory::{ }, db::{api::mutation::AdditionOps, graph::graph::Graph}, io::csv_loader::CsvLoader, + logging::global_info_logger, prelude::*, }; use serde::Deserialize; use std::{env, path::Path, time::Instant}; +use tracing::info; #[derive(Deserialize, std::fmt::Debug)] struct Edge { @@ -17,7 +19,7 @@ struct Edge { fn main() { let now = Instant::now(); - + global_info_logger(); let args: Vec = env::args().collect(); let data_dir = Path::new(args.get(1).expect("No data directory provided")); @@ -39,7 +41,7 @@ fn main() { g }; - println!( + info!( "Loaded graph from encoded data files {} with {} nodes, {} edges which took {} seconds", data_dir.to_str().unwrap(), g.count_nodes(), @@ -51,13 +53,13 @@ fn main() { unweighted_page_rank(&g, Some(100), None, Some(0.00000001), true, None); - println!("PageRank took {} millis", now.elapsed().as_millis()); + info!("PageRank took {} millis", now.elapsed().as_millis()); let now = Instant::now(); weakly_connected_components(&g, 100, None); - println!( + info!( "Connected Components took {} millis", now.elapsed().as_millis() ); diff --git a/python/python/raphtory/__init__.pyi b/python/python/raphtory/__init__.pyi index 0da38b14f5..0a909ff4df 100644 --- a/python/python/raphtory/__init__.pyi +++ b/python/python/raphtory/__init__.pyi @@ -995,7 +995,7 @@ class Edges: class Graph: """A temporal graph.""" - def __init__(self): + def __init__(self, num_shards=None): """Initialize self. See help(type(self)) for accurate signature.""" def add_constant_properties(self, properties): @@ -1881,9 +1881,9 @@ class Graph: embedding, cache=None, overwrite_cache=False, - graph_document=None, - node_document=None, - edge_document=None, + graph_template=None, + node_template=None, + edge_template=None, verbose=False, ): """ @@ -1893,8 +1893,9 @@ class Graph: embedding (Callable[[list], list]): the embedding function to translate documents to embeddings cache (str): the file to be used as a cache to avoid calling the embedding function (optional) overwrite_cache (bool): whether or not to overwrite the cache if there are new embeddings (optional) - node_document (str): the property name to be used as document for nodes (optional) - edge_document (str): the property name to be used as document for edges (optional) + graph_template (str): the document template for the graphs (optional) + node_template (str): the document template for the nodes (optional) + edge_template (str): the document template for the edges (optional) verbose (bool): whether or not to print logs reporting the progress Returns: @@ -4663,9 +4664,9 @@ class PersistentGraph: embedding, cache=None, overwrite_cache=False, - graph_document=None, - node_document=None, - edge_document=None, + graph_template=None, + node_template=None, + edge_template=None, verbose=False, ): """ @@ -4675,8 +4676,9 @@ class PersistentGraph: embedding (Callable[[list], list]): the embedding function to translate documents to embeddings cache (str): the file to be used as a cache to avoid calling the embedding function (optional) overwrite_cache (bool): whether or not to overwrite the cache if there are new embeddings (optional) - node_document (str): the property name to be used as document for nodes (optional) - edge_document (str): the property name to be used as document for edges (optional) + graph_template (str): the document template for the graphs (optional) + node_template (str): the document template for the nodes (optional) + edge_template (str): the document template for the edges (optional) verbose (bool): whether or not to print logs reporting the progress Returns: diff --git a/python/python/raphtory/graphql/__init__.pyi b/python/python/raphtory/graphql/__init__.pyi index 7782ab8470..5f85d9cbb8 100644 --- a/python/python/raphtory/graphql/__init__.pyi +++ b/python/python/raphtory/graphql/__init__.pyi @@ -16,6 +16,10 @@ class GraphServer: cache_capacity=None, cache_tti_seconds=None, log_level=None, + tracing=None, + otlp_agent_host=None, + otlp_agent_port=None, + otlp_tracing_service_name=None, config_path=None, ): """Initialize self. See help(type(self)) for accurate signature.""" @@ -78,9 +82,9 @@ class GraphServer: cache, graph_names=None, embedding=None, - graph_document=None, - node_document=None, - edge_document=None, + graph_template=None, + node_template=None, + edge_template=None, ): """ Vectorise a subset of the graphs of the server. @@ -94,9 +98,9 @@ class GraphServer: graph_names (List[str]): the names of the graphs to vectorise. All by default. cache (str): the directory to use as cache for the embeddings. embedding (Function): the embedding function to translate documents to embeddings. - graph_document (String): the property name to use as the source for the documents on graphs. - node_document (String): the property name to use as the source for the documents on nodes. - edge_document (String): the property name to use as the source for the documents on edges. + graph_template (String): the template to use for graphs. + node_template (String): the template to use for nodes. + edge_template (String): the template to use for edges. Returns: GraphServer: A new server object containing the vectorised graphs. diff --git a/python/python/raphtory/vectors/__init__.pyi b/python/python/raphtory/vectors/__init__.pyi index 7260b72896..c9a5319d91 100644 --- a/python/python/raphtory/vectors/__init__.pyi +++ b/python/python/raphtory/vectors/__init__.pyi @@ -14,6 +14,8 @@ class Document: @property def content(self): ... @property + def embedding(self): ... + @property def entity(self): ... @property def life(self): ... @@ -22,166 +24,60 @@ class VectorisedGraph: def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def append(self, nodes, edges): - """ - Add all the documents from `nodes` and `edges` to the current selection - - Documents added by this call are assumed to have a score of 0. - - Args: - nodes (list): a list of the node ids or nodes to add - edges (list): a list of the edge ids or edges to add - - Returns: - A new vectorised graph containing the updated selection - """ - - def append_by_similarity(self, query, limit, window=None): - """ - Add the top `limit` documents to the current selection using `query` - - Args: - query (str or list): the text or the embedding to score against - limit (int): the maximum number of new documents to add - window ((int | str, int | str)): the window where documents need to belong to in order to be considered - - Returns: - A new vectorised graph containing the updated selection - """ - - def append_edges(self, edges): + def documents_by_similarity(self, query, limit, window=None): """ - Add all the documents from `edges` to the current selection - - Documents added by this call are assumed to have a score of 0. - - Args: - edges (list): a list of the edge ids or edges to add - - Returns: - A new vectorised graph containing the updated selection - """ - - def append_edges_by_similarity(self, query, limit, window=None): - """ - Add the top `limit` edge documents to the current selection using `query` + Search the top scoring documents according to `query` with no more than `limit` documents Args: query (str or list): the text or the embedding to score against - limit (int): the maximum number of new documents to add + limit (int): the maximum number of documents to search window ((int | str, int | str)): the window where documents need to belong to in order to be considered Returns: - A new vectorised graph containing the updated selection - """ - - def append_nodes(self, nodes): - """ - Add all the documents from `nodes` to the current selection - - Documents added by this call are assumed to have a score of 0. - - Args: - nodes (list): a list of the node ids or nodes to add - - Returns: - A new vectorised graph containing the updated selection + The vector selection resulting from the search """ - def append_nodes_by_similarity(self, query, limit, window=None): + def edges_by_similarity(self, query, limit, window=None): """ - Add the top `limit` node documents to the current selection using `query` + Search the top scoring edges according to `query` with no more than `limit` edges Args: query (str or list): the text or the embedding to score against - limit (int): the maximum number of new documents to add + limit (int): the maximum number of new edges to search window ((int | str, int | str)): the window where documents need to belong to in order to be considered Returns: - A new vectorised graph containing the updated selection - """ - - def edges(self): - """Return the edges present in the current selection""" - - def expand(self, hops, window=None): - """ - Add all the documents `hops` hops away to the selection - - Two documents A and B are considered to be 1 hop away of each other if they are on the same - entity or if they are on the same node/edge pair. Provided that, two nodes A and C are n - hops away of each other if there is a document B such that A is n - 1 hops away of B and B - is 1 hop away of C. - - Args: - hops (int): the number of hops to carry out the expansion - window ((int | str, int | str)): the window where documents need to belong to in order to be considered - - Returns: - A new vectorised graph containing the updated selection - """ - - def expand_by_similarity(self, query, limit, window=None): + The vector selection resulting from the search """ - Add the top `limit` adjacent documents with higher score for `query` to the selection - - The expansion algorithm is a loop with two steps on each iteration: - 1. All the documents 1 hop away of some of the documents included on the selection (and - not already selected) are marked as candidates. - 2. Those candidates are added to the selection in descending order according to the - similarity score obtained against the `query`. - This loops goes on until the current selection reaches a total of `limit` documents or - until no more documents are available - - Args: - query (str or list): the text or the embedding to score against - window ((int | str, int | str)): the window where documents need to belong to in order to be considered + def empty_selection(self): + """Return an empty selection of documents""" - Returns: - A new vectorised graph containing the updated selection + def entities_by_similarity(self, query, limit, window=None): """ - - def expand_edges_by_similarity(self, query, limit, window=None): - """ - Add the top `limit` adjacent edge documents with higher score for `query` to the selection - - This function has the same behavior as expand_by_similarity but it only considers edges. + Search the top scoring entities according to `query` with no more than `limit` entities Args: query (str or list): the text or the embedding to score against - limit (int): the maximum number of new documents to add + limit (int): the maximum number of new entities to search window ((int | str, int | str)): the window where documents need to belong to in order to be considered Returns: - A new vectorised graph containing the updated selection + The vector selection resulting from the search """ - def expand_nodes_by_similarity(self, query, limit, window=None): + def nodes_by_similarity(self, query, limit, window=None): """ - Add the top `limit` adjacent node documents with higher score for `query` to the selection - - This function has the same behavior as expand_by_similarity but it only considers nodes. + Search the top scoring nodes according to `query` with no more than `limit` nodes Args: query (str or list): the text or the embedding to score against - limit (int): the maximum number of new documents to add + limit (int): the maximum number of new nodes to search window ((int | str, int | str)): the window where documents need to belong to in order to be considered Returns: - A new vectorised graph containing the updated selection + The vector selection resulting from the search """ - def get_documents(self): - """Return the documents present in the current selection""" - - def get_documents_with_scores(self): - """Return the documents alongside their scores present in the current selection""" - - def nodes(self): - """Return the nodes present in the current selection""" - def save_embeddings(self, file): """Save the embeddings present in this graph to `file` so they can be further used in a call to `vectorise`""" - -def generate_property_list(entity, filter_out=..., force_static=...): ... diff --git a/python/tests/graphql/edit_graph/test_deprecated_save.py b/python/tests/graphql/edit_graph/test_deprecated_save.py index 6bcb04cd51..f40fe99460 100644 --- a/python/tests/graphql/edit_graph/test_deprecated_save.py +++ b/python/tests/graphql/edit_graph/test_deprecated_save.py @@ -1055,7 +1055,9 @@ def test_archive_graph_succeeds(): ) } }""" - assert client.query(update_archive_graph) == {'updateGraph': {'updateConstantProperties': True}} + assert client.query(update_archive_graph) == { + "updateGraph": {"updateConstantProperties": True} + } assert ( client.query(query_is_archive)["graph"]["properties"]["constant"]["get"][ "value" @@ -1069,7 +1071,9 @@ def test_archive_graph_succeeds(): ) } }""" - assert client.query(update_archive_graph) == {'updateGraph': {'updateConstantProperties': True}} + assert client.query(update_archive_graph) == { + "updateGraph": {"updateConstantProperties": True} + } assert ( client.query(query_is_archive)["graph"]["properties"]["constant"]["get"][ "value" @@ -1103,7 +1107,9 @@ def test_archive_graph_succeeds_at_namespace(): ) } }""" - assert client.query(update_archive_graph) == {'updateGraph': {'updateConstantProperties': True}} + assert client.query(update_archive_graph) == { + "updateGraph": {"updateConstantProperties": True} + } assert ( client.query(query_is_archive)["graph"]["properties"]["constant"]["get"][ "value" @@ -1117,7 +1123,9 @@ def test_archive_graph_succeeds_at_namespace(): ) } }""" - assert client.query(update_archive_graph) == {'updateGraph': {'updateConstantProperties': True}} + assert client.query(update_archive_graph) == { + "updateGraph": {"updateConstantProperties": True} + } assert ( client.query(query_is_archive)["graph"]["properties"]["constant"]["get"][ "value" diff --git a/python/tests/graphql/misc/test_tracing.py b/python/tests/graphql/misc/test_tracing.py new file mode 100644 index 0000000000..f870d20e68 --- /dev/null +++ b/python/tests/graphql/misc/test_tracing.py @@ -0,0 +1,38 @@ +import tempfile + +from raphtory import Graph +from raphtory.graphql import GraphServer, RaphtoryClient + + +# TODO this doesn't work currently due to how we terminate with Tracing +def test_server_start_on_default_port(): + g = Graph() + g.add_edge(1, "ben", "hamza") + g.add_edge(2, "haaroon", "hamza") + g.add_edge(3, "ben", "haaroon") + + tmp_work_dir = tempfile.mkdtemp() + with GraphServer(tmp_work_dir, tracing=True).start(): + client = RaphtoryClient("http://localhost:1736") + client.send_graph(path="g", graph=g) + + query = """{graph(path: "g") {nodes {list {name}}}}""" + assert client.query(query) == { + "graph": { + "nodes": { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + } + } + } + with GraphServer(tmp_work_dir, tracing=True).start(): + client = RaphtoryClient("http://localhost:1736") + client.send_graph(path="g2", graph=g) + + query = """{graph(path: "g2") {nodes {list {name}}}}""" + assert client.query(query) == { + "graph": { + "nodes": { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + } + } + } diff --git a/python/tests/graphql/test_graph_file_time_stats.py b/python/tests/graphql/test_graph_file_time_stats.py index 8ac267e21c..ea77293be5 100644 --- a/python/tests/graphql/test_graph_file_time_stats.py +++ b/python/tests/graphql/test_graph_file_time_stats.py @@ -15,22 +15,28 @@ def test_graph_file_time_stats(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) graph_file_path = os.path.join(work_dir, "shivam", "g3") g.save_to_file(graph_file_path) - + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """{graph(path: "shivam/g3") { created lastOpened lastUpdated }}""" result = client.query(query) - - gql_created_time = result['graph']['created'] - gql_last_opened_time = result['graph']['lastOpened'] - gql_last_updated_time = result['graph']['lastUpdated'] - + + gql_created_time = result["graph"]["created"] + gql_last_opened_time = result["graph"]["lastOpened"] + gql_last_updated_time = result["graph"]["lastUpdated"] + file_stats = os.stat(graph_file_path) created_time_fs = file_stats.st_ctime * 1000 last_opened_time_fs = file_stats.st_atime * 1000 last_updated_time_fs = file_stats.st_mtime * 1000 - assert abs(gql_created_time - created_time_fs) < 1000, f"Mismatch in created time: FS({created_time_fs}) vs GQL({gql_created_time})" - assert abs(gql_last_opened_time - last_opened_time_fs) < 1000, f"Mismatch in last opened time: FS({last_opened_time_fs}) vs GQL({gql_last_opened_time})" - assert abs(gql_last_updated_time - last_updated_time_fs) < 1000, f"Mismatch in last updated time: FS({last_updated_time_fs}) vs GQL({gql_last_updated_time})" + assert ( + abs(gql_created_time - created_time_fs) < 1000 + ), f"Mismatch in created time: FS({created_time_fs}) vs GQL({gql_created_time})" + assert ( + abs(gql_last_opened_time - last_opened_time_fs) < 1000 + ), f"Mismatch in last opened time: FS({last_opened_time_fs}) vs GQL({gql_last_opened_time})" + assert ( + abs(gql_last_updated_time - last_updated_time_fs) < 1000 + ), f"Mismatch in last updated time: FS({last_updated_time_fs}) vs GQL({gql_last_updated_time})" diff --git a/python/tests/test_loaders/test_load_from_pandas.py b/python/tests/test_loaders/test_load_from_pandas.py index 55d9da81e4..8001e332f0 100644 --- a/python/tests/test_loaders/test_load_from_pandas.py +++ b/python/tests/test_loaders/test_load_from_pandas.py @@ -72,8 +72,8 @@ def test_load_from_pandas_with_invalid_data(): def assertions(exc_info): assert "ArrowInvalid" in str(exc_info.value) assert ( - "Could not convert '3.0 KG' with type str: tried to convert to double" - in str(exc_info.value) + "Could not convert '3.0 KG' with type str: tried to convert to double" + in str(exc_info.value) ) # Use pytest.raises to expect an exception @@ -90,8 +90,8 @@ def assertions(exc_info): # Optionally, you can check the exception message or type assert "ArrowInvalid" in str(exc_info.value) assert ( - "Could not convert '3.0 KG' with type str: tried to convert to double" - in str(exc_info.value) + "Could not convert '3.0 KG' with type str: tried to convert to double" + in str(exc_info.value) ) @@ -386,7 +386,9 @@ def assertions3(g): {"test_layer": "test_tag"}, {"test_layer": "test_tag"}, ] - assert dict(zip(g.edges.id, g.edges.properties.constant.get("marbles_const"))) == { + assert dict( + zip(g.edges.id, g.edges.properties.constant.get("marbles_const")) + ) == { (1, 2): {"test_layer": "red"}, (2, 3): {"test_layer": "blue"}, (3, 4): {"test_layer": "green"}, @@ -455,7 +457,13 @@ def assertions5(g): "Person", "Person", ] - assert set(g.layers(["test_layer"]).edges.id) == {(1, 2), (2, 3), (3, 4), (4, 5), (5, 6)} + assert set(g.layers(["test_layer"]).edges.id) == { + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (5, 6), + } g = Graph() g.load_edges_from_pandas( @@ -572,15 +580,18 @@ def assertions7(g): assertions7(g) def assertions8(g): - assert dict(zip(g.layers( - ["layer 1", "layer 2", "layer 3"] - ).edges.id, g.layers( - ["layer 1", "layer 2", "layer 3"] - ).edges.properties.constant.get("marbles_const"))) == { - (1, 2): {"layer 1": "red"}, - (2, 3): {"layer 2": "blue"}, - (3, 4): {"layer 3": "green"}, - } + assert dict( + zip( + g.layers(["layer 1", "layer 2", "layer 3"]).edges.id, + g.layers( + ["layer 1", "layer 2", "layer 3"] + ).edges.properties.constant.get("marbles_const"), + ) + ) == { + (1, 2): {"layer 1": "red"}, + (2, 3): {"layer 2": "blue"}, + (3, 4): {"layer 3": "green"}, + } assert dict(zip(g.edges.id, g.edges.properties.constant.get("tag"))) == { (1, 2): {"layer 1": "test_tag"}, (2, 3): {"layer 2": "test_tag"}, @@ -640,10 +651,8 @@ def assertions_layers_in_df(g): assert g.layers(["layer 1"]).edges.src.id.collect() == [1] assert g.layers(["layer 3"]).edges.src.id.collect() == [3] with pytest.raises( - Exception, - match=re.escape( - "Invalid layer: test_layer" - ), + Exception, + match=re.escape("Invalid layer: test_layer"), ): g.layers(["test_layer"]) @@ -694,10 +703,10 @@ def test_missing_columns(): ) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: not_src, not_dst, not_time" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: not_src, not_dst, not_time" + ), ): g = Graph() g.load_edges_from_pandas( @@ -708,10 +717,10 @@ def test_missing_columns(): ) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: not_src, not_dst, not_time" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: not_src, not_dst, not_time" + ), ): g = PersistentGraph() g.load_edges_from_pandas( @@ -722,10 +731,10 @@ def test_missing_columns(): ) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: not_weight, bleep_bloop" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: not_weight, bleep_bloop" + ), ): g = Graph() g.load_edges_from_pandas( @@ -739,10 +748,10 @@ def test_missing_columns(): g.load_nodes_from_pandas(df=nodes_df, time="time", id="id", properties=["name"]) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: not_weight, bleep_bloop" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: not_weight, bleep_bloop" + ), ): g = PersistentGraph() g.load_edges_from_pandas( @@ -756,10 +765,10 @@ def test_missing_columns(): g.load_nodes_from_pandas(df=nodes_df, time="time", id="id", properties=["name"]) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: not_id, not_time, not_name" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: not_id, not_time, not_name" + ), ): g = Graph() g.load_edges_from_pandas( @@ -774,10 +783,10 @@ def test_missing_columns(): ) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: not_id, not_time, not_name" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: not_id, not_time, not_name" + ), ): g = PersistentGraph() g.load_edges_from_pandas( @@ -792,10 +801,10 @@ def test_missing_columns(): ) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: sauce, dist, wait, marples" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: sauce, dist, wait, marples" + ), ): g = Graph() g.load_edge_props_from_pandas( @@ -806,10 +815,10 @@ def test_missing_columns(): ) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: sauce, dist, wait, marples" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: sauce, dist, wait, marples" + ), ): g = PersistentGraph() g.load_edge_props_from_pandas( @@ -820,10 +829,10 @@ def test_missing_columns(): ) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: sauce, wait, marples" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: sauce, wait, marples" + ), ): g = Graph() g.load_node_props_from_pandas( @@ -833,10 +842,10 @@ def test_missing_columns(): ) with pytest.raises( - Exception, - match=re.escape( - "columns are not present within the dataframe: sauce, wait, marples" - ), + Exception, + match=re.escape( + "columns are not present within the dataframe: sauce, wait, marples" + ), ): g = PersistentGraph() g.load_node_props_from_pandas( @@ -851,13 +860,13 @@ def test_none_columns_edges(): {"src": [1, None, 3, 4, 5], "dst": [2, 3, 4, 5, 6], "time": [1, 2, 3, 4, 5]} ) with pytest.raises( - Exception, match=re.escape("Float64 not supported as node id type") + Exception, match=re.escape("Float64 not supported as node id type") ): g = Graph() g.load_edges_from_pandas(edges_df, "time", "src", "dst") with pytest.raises( - Exception, match=re.escape("Float64 not supported as node id type") + Exception, match=re.escape("Float64 not supported as node id type") ): PersistentGraph().load_edges_from_pandas(edges_df, "time", "src", "dst") @@ -865,11 +874,11 @@ def test_none_columns_edges(): {"src": [1, 2, 3, 4, 5], "dst": [2, 3, 4, None, 6], "time": [1, 2, 3, 4, 5]} ) with pytest.raises( - Exception, match=re.escape("Float64 not supported as node id type") + Exception, match=re.escape("Float64 not supported as node id type") ): Graph().load_edges_from_pandas(edges_df, "time", "src", "dst") with pytest.raises( - Exception, match=re.escape("Float64 not supported as node id type") + Exception, match=re.escape("Float64 not supported as node id type") ): PersistentGraph().load_edges_from_pandas(edges_df, "time", "src", "dst") @@ -877,11 +886,11 @@ def test_none_columns_edges(): {"src": [1, 2, 3, 4, 5], "dst": [2, 3, 4, 5, 6], "time": [1, 2, None, 4, 5]} ) with pytest.raises( - Exception, match=re.escape("Float64 not supported for time column") + Exception, match=re.escape("Float64 not supported for time column") ): Graph().load_edges_from_pandas(edges_df, "time", "src", "dst") with pytest.raises( - Exception, match=re.escape("Float64 not supported for time column") + Exception, match=re.escape("Float64 not supported for time column") ): PersistentGraph().load_edges_from_pandas(edges_df, "time", "src", "dst") @@ -939,10 +948,10 @@ def test_unparsable_props(): ) with pytest.raises( - Exception, - match=re.escape( - """"Could not convert '2.0' with type str: tried to convert to double", 'Conversion failed for column weight with type object'""" - ), + Exception, + match=re.escape( + """"Could not convert '2.0' with type str: tried to convert to double", 'Conversion failed for column weight with type object'""" + ), ): Graph().load_edges_from_pandas( edges_df, @@ -952,10 +961,10 @@ def test_unparsable_props(): properties=["weight"], ) with pytest.raises( - Exception, - match=re.escape( - """"Could not convert '2.0' with type str: tried to convert to double", 'Conversion failed for column weight with type object'""" - ), + Exception, + match=re.escape( + """"Could not convert '2.0' with type str: tried to convert to double", 'Conversion failed for column weight with type object'""" + ), ): PersistentGraph().load_edges_from_pandas( edges_df, @@ -1083,16 +1092,16 @@ def test_edge_both_option_failures_pandas(): # CHECK ALL EDGE FUNCTIONS ON GRAPH FAIL WITH BOTH LAYER AND LAYER_COL g = Graph() with pytest.raises( - Exception, - match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", + Exception, + match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", ): g.load_edges_from_pandas( edges_df, "time", "src", "dst", layer="blah", layer_col="marbles" ) with pytest.raises( - Exception, - match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", + Exception, + match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", ): g.load_edge_props_from_pandas( edges_df, "src", "dst", layer="blah", layer_col="marbles" @@ -1123,7 +1132,9 @@ def test_edge_both_option_failures_pandas(): ["blah"], ] assert g.unique_layers == ["_default", "blah"] - assert dict(zip(g.layer("blah").edges.id, g.layer("blah").edges.properties.get("marbles"))) == { + assert dict( + zip(g.layer("blah").edges.id, g.layer("blah").edges.properties.get("marbles")) + ) == { (1, 2): "red", (2, 3): "blue", (3, 4): "green", @@ -1141,7 +1152,14 @@ def test_edge_both_option_failures_pandas(): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } g = Graph() g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer_col="marbles") @@ -1155,7 +1173,14 @@ def test_edge_both_option_failures_pandas(): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } assert dict(zip(g.edges.id, g.edges.properties.get("marbles"))) == { (1, 2): {"red": "red"}, (2, 3): {"blue": "blue"}, @@ -1166,24 +1191,24 @@ def test_edge_both_option_failures_pandas(): g = PersistentGraph() with pytest.raises( - Exception, - match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", + Exception, + match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", ): g.load_edges_from_pandas( edges_df, "time", "src", "dst", layer="blah", layer_col="marbles" ) with pytest.raises( - Exception, - match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", + Exception, + match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", ): g.load_edge_props_from_pandas( edges_df, "src", "dst", layer="blah", layer_col="marbles" ) with pytest.raises( - Exception, - match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", + Exception, + match=r"You cannot set ‘layer_name’ and ‘layer_col’ at the same time. Please pick one or the other.", ): g.load_edge_deletions_from_pandas( edges_df, "time", "src", "dst", layer="blah", layer_col="marbles" @@ -1214,7 +1239,9 @@ def test_edge_both_option_failures_pandas(): ["blah"], ] assert g.unique_layers == ["_default", "blah"] - assert dict(zip(g.layer("blah").edges.id, g.layer("blah").edges.properties.get("marbles"))) == { + assert dict( + zip(g.layer("blah").edges.id, g.layer("blah").edges.properties.get("marbles")) + ) == { (1, 2): "red", (2, 3): "blue", (3, 4): "green", @@ -1243,7 +1270,14 @@ def test_edge_both_option_failures_pandas(): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } g = PersistentGraph() g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer_col="marbles") @@ -1257,7 +1291,14 @@ def test_edge_both_option_failures_pandas(): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } assert dict(zip(g.edges.id, g.edges.properties.get("marbles"))) == { (1, 2): {"red": "red"}, (2, 3): {"blue": "blue"}, @@ -1277,7 +1318,14 @@ def test_edge_both_option_failures_pandas(): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } def test_node_both_option_failures_pandas(): @@ -1291,8 +1339,8 @@ def test_node_both_option_failures_pandas(): ) # CHECK ALL NODE FUNCTIONS ON GRAPH FAIL WITH BOTH NODE_TYPE AND NODE_TYPE_COL with pytest.raises( - Exception, - match=r"You cannot set ‘node_type_name’ and ‘node_type_col’ at the same time. Please pick one or the other.", + Exception, + match=r"You cannot set ‘node_type_name’ and ‘node_type_col’ at the same time. Please pick one or the other.", ): g = Graph() g.load_nodes_from_pandas( @@ -1300,8 +1348,8 @@ def test_node_both_option_failures_pandas(): ) with pytest.raises( - Exception, - match=r"You cannot set ‘node_type_name’ and ‘node_type_col’ at the same time. Please pick one or the other.", + Exception, + match=r"You cannot set ‘node_type_name’ and ‘node_type_col’ at the same time. Please pick one or the other.", ): g = Graph() g.load_node_props_from_pandas( diff --git a/python/tests/test_loaders/test_load_from_parquet.py b/python/tests/test_loaders/test_load_from_parquet.py index e419153d42..4ab8c36b3d 100644 --- a/python/tests/test_loaders/test_load_from_parquet.py +++ b/python/tests/test_loaders/test_load_from_parquet.py @@ -155,14 +155,18 @@ def assert_expected_node_property_dept(g): def assert_expected_edge_properties(g): - assert dict(zip(g.layers(["layer 1", "layer 2", "layer 3"]).edges.id, - g.layers(["layer 1", "layer 2", "layer 3"]).edges.properties.constant.get( - "marbles_const" - ))) == { - (1, 2): {"layer 1": "red"}, - (2, 3): {"layer 2": "blue"}, - (3, 4): {"layer 3": "green"}, - } + assert dict( + zip( + g.layers(["layer 1", "layer 2", "layer 3"]).edges.id, + g.layers(["layer 1", "layer 2", "layer 3"]).edges.properties.constant.get( + "marbles_const" + ), + ) + ) == { + (1, 2): {"layer 1": "red"}, + (2, 3): {"layer 2": "blue"}, + (3, 4): {"layer 3": "green"}, + } assert dict(zip(g.edges.id, g.edges.properties.constant.get("tag"))) == { (1, 2): {"layer 1": "test_tag"}, (2, 3): {"layer 2": "test_tag"}, @@ -218,10 +222,8 @@ def assert_expected_layers(g): 5, ] with pytest.raises( - Exception, - match=re.escape( - "Invalid layer: test_layer" - ), + Exception, + match=re.escape("Invalid layer: test_layer"), ): g.layers(["test_layer"]) @@ -512,8 +514,8 @@ def test_edge_both_option_failures_parquet(parquet_files): # CHECK ALL EDGE FUNCTIONS ON GRAPH FAIL WITH BOTH LAYER AND LAYER_COL g = Graph() with pytest.raises( - Exception, - match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", ): g.load_edges_from_parquet( edges_parquet_file_path, @@ -525,8 +527,8 @@ def test_edge_both_option_failures_parquet(parquet_files): ) with pytest.raises( - Exception, - match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", ): g.load_edge_props_from_parquet( edges_parquet_file_path, "src", "dst", layer="blah", layer_col="marbles" @@ -565,7 +567,9 @@ def test_edge_both_option_failures_parquet(parquet_files): ["blah"], ] assert g.unique_layers == ["_default", "blah"] - assert dict(zip(g.layer("blah").edges.id, g.layer("blah").edges.properties.get("marbles"))) == { + assert dict( + zip(g.layer("blah").edges.id, g.layer("blah").edges.properties.get("marbles")) + ) == { (1, 2): "red", (2, 3): "blue", (3, 4): "green", @@ -585,7 +589,14 @@ def test_edge_both_option_failures_parquet(parquet_files): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } g = Graph() g.load_edges_from_parquet( @@ -605,7 +616,14 @@ def test_edge_both_option_failures_parquet(parquet_files): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } assert dict(zip(g.edges.id, g.edges.properties.get("marbles"))) == { (1, 2): {"red": "red"}, (2, 3): {"blue": "blue"}, @@ -616,8 +634,8 @@ def test_edge_both_option_failures_parquet(parquet_files): g = PersistentGraph() with pytest.raises( - Exception, - match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", ): g.load_edges_from_parquet( edges_parquet_file_path, @@ -629,16 +647,16 @@ def test_edge_both_option_failures_parquet(parquet_files): ) with pytest.raises( - Exception, - match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", ): g.load_edge_props_from_parquet( edges_parquet_file_path, "src", "dst", layer="blah", layer_col="marbles" ) with pytest.raises( - Exception, - match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", ): g.load_edge_deletions_from_parquet( edges_parquet_file_path, @@ -682,7 +700,9 @@ def test_edge_both_option_failures_parquet(parquet_files): ["blah"], ] assert g.unique_layers == ["_default", "blah"] - assert dict(zip(g.layer("blah").edges.id, g.layer("blah").edges.properties.get("marbles"))) == { + assert dict( + zip(g.layer("blah").edges.id, g.layer("blah").edges.properties.get("marbles")) + ) == { (1, 2): "red", (2, 3): "blue", (3, 4): "green", @@ -715,7 +735,14 @@ def test_edge_both_option_failures_parquet(parquet_files): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } g = PersistentGraph() g.load_edges_from_parquet( @@ -735,7 +762,14 @@ def test_edge_both_option_failures_parquet(parquet_files): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } assert dict(zip(g.edges.id, g.edges.properties.get("marbles"))) == { (1, 2): {"red": "red"}, (2, 3): {"blue": "blue"}, @@ -755,7 +789,14 @@ def test_edge_both_option_failures_parquet(parquet_files): (4, 5): ["yellow"], (5, 6): ["purple"], } - assert set(g.unique_layers) == {"_default", "red", "blue", "green", "yellow", "purple"} + assert set(g.unique_layers) == { + "_default", + "red", + "blue", + "green", + "yellow", + "purple", + } def test_node_both_option_failures_parquet(parquet_files): @@ -767,9 +808,10 @@ def test_node_both_option_failures_parquet(parquet_files): # CHECK ALL NODE FUNCTIONS ON GRAPH FAIL WITH BOTH NODE_TYPE AND NODE_TYPE_COL with pytest.raises( - Exception, - match=re.escape( - r'Failed to load graph: Failed to load graph WrongNumOfArgs("node_type_name", "node_type_col")'), + Exception, + match=re.escape( + r'Failed to load graph: Failed to load graph WrongNumOfArgs("node_type_name", "node_type_col")' + ), ): g = Graph() g.load_nodes_from_parquet( @@ -781,9 +823,10 @@ def test_node_both_option_failures_parquet(parquet_files): ) with pytest.raises( - Exception, - match=re.escape( - r'Failed to load graph: Failed to load graph WrongNumOfArgs("node_type_name", "node_type_col")'), + Exception, + match=re.escape( + r'Failed to load graph: Failed to load graph WrongNumOfArgs("node_type_name", "node_type_col")' + ), ): g = Graph() g.load_node_props_from_parquet( diff --git a/python/tests/test_vectors.py b/python/tests/test_vectors.py index d027d1a956..6662a13763 100644 --- a/python/tests/test_vectors.py +++ b/python/tests/test_vectors.py @@ -48,7 +48,9 @@ def create_graph() -> VectorisedGraph: g.add_edge(3, "node1", "node3", {"name": "edge2"}) g.add_edge(4, "node3", "node4", {"name": "edge3"}) - vg = g.vectorise(embedding, node_template="{{ name }}", edge_template="{{ props.name }}") + vg = g.vectorise( + embedding, node_template="{{ name }}", edge_template="{{ props.name }}" + ) return vg @@ -120,10 +122,14 @@ def test_search(): assert [doc.content for doc in docs] == ["node3", "edge3", "edge2"] # chained search - node_selection = vg.nodes_by_similarity("node2", 1); - edge_selection = vg.edges_by_similarity("node3", 1); - entity_selection = vg.entities_by_similarity("node1", 4); - docs = node_selection.append(edge_selection).append(entity_selection).get_documents()[:4] + node_selection = vg.nodes_by_similarity("node2", 1) + edge_selection = vg.edges_by_similarity("node3", 1) + entity_selection = vg.entities_by_similarity("node1", 4) + docs = ( + node_selection.append(edge_selection) + .append(entity_selection) + .get_documents()[:4] + ) assert [doc.content for doc in docs] == ["node2", "edge3", "node1", "edge1"] # the intention of this test was getting all the documents of for different entities, # including at least node and one edge at the top. diff --git a/raphtory-api/Cargo.toml b/raphtory-api/Cargo.toml index 7109b4d4f1..a7eda04328 100644 --- a/raphtory-api/Cargo.toml +++ b/raphtory-api/Cargo.toml @@ -28,7 +28,8 @@ rand = { workspace = true } quickcheck_macros = { workspace = true } num-traits = { workspace = true } twox-hash.workspace = true - +tracing-subscriber = {workspace = true} +tracing = {workspace = true} [dev-dependencies] proptest.workspace = true quickcheck.workspace = true diff --git a/raphtory-api/src/core/utils/logging.rs b/raphtory-api/src/core/utils/logging.rs new file mode 100644 index 0000000000..5c82fe490f --- /dev/null +++ b/raphtory-api/src/core/utils/logging.rs @@ -0,0 +1,41 @@ +use std::sync::OnceLock; +use tracing_subscriber::{ + fmt, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry, +}; + +pub fn get_log_env(log_level: String) -> EnvFilter { + EnvFilter::new(format!( + "pometry-storage={},pometry-storage-private={},raphtory={},raphtory-api={},raphtory-benchmark={},raphtory-cypher={},raphtory-graphql={}", + log_level, log_level, log_level, log_level, log_level, log_level, log_level + )) +} + +pub fn init_global_logger(log_level: String) { + static INIT: OnceLock<()> = OnceLock::new(); + + INIT.get_or_init(|| { + let filter = get_log_env(log_level); + let registry = Registry::default() + .with(filter) + .with(fmt::layer().pretty().with_span_events(FmtSpan::NONE)); + registry.try_init().ok(); + }); +} + +pub fn global_error_logger() { + init_global_logger("ERROR".to_string()) +} +pub fn global_warn_logger() { + init_global_logger("WARN".to_string()) +} + +pub fn global_info_logger() { + init_global_logger("INFO".to_string()) +} + +pub fn global_debug_logger() { + init_global_logger("DEBUG".to_string()) +} +pub fn global_trace_logger() { + init_global_logger("TRACE".to_string()) +} diff --git a/raphtory-api/src/core/utils/mod.rs b/raphtory-api/src/core/utils/mod.rs index 62c68590b8..9f0346076f 100644 --- a/raphtory-api/src/core/utils/mod.rs +++ b/raphtory-api/src/core/utils/mod.rs @@ -1 +1,2 @@ pub mod hashing; +pub mod logging; diff --git a/raphtory-benchmark/Cargo.toml b/raphtory-benchmark/Cargo.toml index 8aa8c0bf8a..9c052dba1c 100644 --- a/raphtory-benchmark/Cargo.toml +++ b/raphtory-benchmark/Cargo.toml @@ -17,6 +17,7 @@ tempfile = { workspace = true } clap = { workspace = true } csv = { workspace = true } flate2 = { workspace = true } +tracing = {workspace = true} [[bench]] name = "tgraph_benchmarks" diff --git a/raphtory-benchmark/bin/main.rs b/raphtory-benchmark/bin/main.rs index 0274fbedad..fb735c89ca 100644 --- a/raphtory-benchmark/bin/main.rs +++ b/raphtory-benchmark/bin/main.rs @@ -9,12 +9,14 @@ use raphtory::{ io::csv_loader::CsvLoader, prelude::{AdditionOps, Graph, GraphViewOps, NodeViewOps, NO_PROPS}, }; +use raphtory_api::core::utils::logging::global_debug_logger; use std::{ fs::File, io::{self, Read, Write}, path::Path, time::Instant, }; +use tracing::{debug, info}; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -57,7 +59,8 @@ struct Args { } fn main() { - println!( + global_debug_logger(); + info!( " ██████╗ ███████╗███╗ ██╗ ██████╗██╗ ██╗███╗ ███╗ █████╗ ██████╗ ██╗ ██╗ ██╔══██╗██╔════╝████╗ ██║██╔════╝██║ ██║████╗ ████║██╔══██╗██╔══██╗██║ ██╔╝ @@ -71,7 +74,7 @@ fn main() { // Set default values let debug = args.debug; if debug { - println!( + info!( " .___ .____ ____ . . ___ __ __ ___ .___ .____ / ` / / \\ / / .' \\ | | .' `. / ` / @@ -80,7 +83,7 @@ fn main() { /---/ /----/ `----' `._.' `.___| / / `.__.' /---/ /----/ " ); - println!("Debug mode enabled.\nArguments: {:?}", args); + info!("Debug mode enabled.\nArguments: {:?}", args); } let header = args.header; let delimiter = args.delimiter; @@ -93,12 +96,12 @@ fn main() { if download { let url = "https://osf.io/download/nbq6h/"; - println!("Downloading default file from url {}...", url); + info!("Downloading default file from url {}...", url); // make err msg from url and custom string let err_msg = format!("Failed to download file from {}", url); let path = fetch_file("simple-relationships.csv.gz", true, url, 1200).expect(&err_msg); - println!("Downloaded file to {}", path.to_str().unwrap()); - println!("Unpacking file..."); + info!("Downloaded file to {}", path.to_str().unwrap()); + info!("Unpacking file..."); // extract a file from a gz archive // Open the input .csv.gz file let input_file = File::open(&path).expect("Failed to open downloaded file"); @@ -124,29 +127,29 @@ fn main() { } // exit program - println!("Downloaded+Unpacked file, please run again without --download flag and with --file-path={}", dst_file); + info!("Downloaded+Unpacked file, please run again without --download flag and with --file-path={}", dst_file); return; } if file_path.is_empty() { - println!("You did not set a file path"); + info!("You did not set a file path"); return; } if !Path::new(&file_path).exists() { - println!("File path does not exist or is not a file {}", &file_path); + info!("File path does not exist or is not a file {}", &file_path); return; } if debug { - println!("Reading file {}", &file_path); + debug!("Reading file {}", &file_path); } - println!("Running setup..."); + info!("Running setup..."); let mut now = Instant::now(); // Iterate over the CSV records let g = match num_shards { Some(num_shards) => { - println!("Constructing graph with {num_shards} shards."); + info!("Constructing graph with {num_shards} shards."); Graph::new_with_shards(num_shards) } None => Graph::new(), @@ -167,16 +170,17 @@ fn main() { 1i64 }; if debug { - println!("Adding edge {} -> {} at time {}", src_id, dst_id, edge_time); + debug!("Adding edge {} -> {} at time {}", src_id, dst_id, edge_time); } g.add_edge(edge_time, src_id, dst_id, NO_PROPS, None) .expect("Failed to add edge"); }) .expect("Failed to load graph from CSV data files"); - println!("Setup took {} seconds", now.elapsed().as_secs_f64()); + info!("Setup took {} seconds", now.elapsed().as_secs_f64()); if debug { - println!( + //leaving this here for now, need to remove the flag and handle properly + debug!( "Graph has {} nodes and {} edges", g.count_nodes(), g.count_edges() @@ -186,23 +190,23 @@ fn main() { // Degree of all nodes now = Instant::now(); let _degree = g.nodes().iter().map(|v| v.degree()).collect::>(); - println!("Degree: {} seconds", now.elapsed().as_secs_f64()); + info!("Degree: {} seconds", now.elapsed().as_secs_f64()); // Out neighbours of all nodes with time now = Instant::now(); let nodes = &g.nodes(); let _out_neighbours = nodes.iter().map(|v| v.out_neighbours()).collect::>(); - println!("Out neighbours: {} seconds", now.elapsed().as_secs_f64()); + info!("Out neighbours: {} seconds", now.elapsed().as_secs_f64()); // page rank with time now = Instant::now(); let _page_rank = unweighted_page_rank(&g, Some(1000), None, None, true, None); - println!("Page rank: {} seconds", now.elapsed().as_secs_f64()); + info!("Page rank: {} seconds", now.elapsed().as_secs_f64()); // connected community_detection with time now = Instant::now(); let _cc = weakly_connected_components(&g, usize::MAX, None); - println!( + info!( "Connected community_detection: {} seconds", now.elapsed().as_secs_f64() ); diff --git a/raphtory-benchmark/src/common/mod.rs b/raphtory-benchmark/src/common/mod.rs index 5a21c49513..78d7ddbde4 100644 --- a/raphtory-benchmark/src/common/mod.rs +++ b/raphtory-benchmark/src/common/mod.rs @@ -5,8 +5,10 @@ use criterion::{ }; use rand::{distributions::Uniform, seq::*, Rng, SeedableRng}; use raphtory::{db::api::view::StaticGraphViewOps, prelude::*}; +use raphtory_api::core::utils::logging::global_info_logger; use std::collections::HashSet; use tempfile::NamedTempFile; +use tracing::info; fn make_index_gen() -> Box> { let rng = rand::thread_rng(); @@ -268,8 +270,9 @@ pub fn run_analysis_benchmarks( F: Fn() -> G, G: StaticGraphViewOps, { + global_info_logger(); let graph = make_graph(); - println!( + info!( "Num layers {:?}, node count: {}, edge_count: {}", graph.unique_layers().count(), graph.count_nodes(), diff --git a/raphtory-cypher/Cargo.toml b/raphtory-cypher/Cargo.toml index 52dfd956f4..88a030dc83 100644 --- a/raphtory-cypher/Cargo.toml +++ b/raphtory-cypher/Cargo.toml @@ -37,6 +37,7 @@ thiserror.workspace = true serde.workspace = true serde_json.workspace = true itertools.workspace = true +tracing.workspace = true [dev-dependencies] proptest.workspace = true diff --git a/raphtory-cypher/examples/raphtory_cypher.rs b/raphtory-cypher/examples/raphtory_cypher.rs index 4633755fb9..8359add122 100644 --- a/raphtory-cypher/examples/raphtory_cypher.rs +++ b/raphtory-cypher/examples/raphtory_cypher.rs @@ -17,9 +17,13 @@ mod cypher { use arrow::util::pretty::print_batches; use clap::Parser; use futures::{stream, StreamExt}; - use raphtory::disk_graph::{graph_impl::ParquetLayerCols, DiskGraphStorage}; + use raphtory::{ + disk_graph::{graph_impl::ParquetLayerCols, DiskGraphStorage}, + logging::global_info_logger, + }; use raphtory_cypher::{run_cypher, run_cypher_to_streams, run_sql}; use serde::{de::DeserializeOwned, Deserialize}; + use tracing::info; /// Query graph with cypher #[derive(Parser, Debug)] @@ -133,6 +137,7 @@ mod cypher { // #[tokio::main] pub async fn main() { + global_info_logger(); let args = Args::parse(); match args { @@ -151,7 +156,8 @@ mod cypher { let now = std::time::Instant::now(); let batches = df.collect().await.unwrap(); - println!("Query execution time: {:?}", now.elapsed()); + global_info_logger(); + info!("Query execution time: {:?}", now.elapsed()); print_batches(&batches).expect("Failed to print batches"); } else { let streams = run_cypher_to_streams(&args.query, &graph).await.unwrap(); @@ -161,7 +167,7 @@ mod cypher { .map(|batch| batch.num_rows()) .fold(0, |acc, x| async move { acc + x }) .await; - println!("Query execution: {:?}, num_rows: {num_rows}", now.elapsed()); + info!("Query execution: {:?}, num_rows: {num_rows}", now.elapsed()); } } diff --git a/raphtory-cypher/src/executor/table_provider/edge.rs b/raphtory-cypher/src/executor/table_provider/edge.rs index b32fff2308..089c17ca31 100644 --- a/raphtory-cypher/src/executor/table_provider/edge.rs +++ b/raphtory-cypher/src/executor/table_provider/edge.rs @@ -428,10 +428,13 @@ mod test { use super::*; use arrow::util::pretty::print_batches; use datafusion::execution::context::SessionContext; + use raphtory::logging::global_info_logger; use tempfile::tempdir; + use tracing::info; #[tokio::test] async fn test_edge_list_table_provider() { + global_info_logger(); let ctx = SessionContext::new(); let graph_dir = tempdir().unwrap(); let edges = vec![ @@ -458,7 +461,7 @@ mod test { "with a AS (select * from graph), b as (select * from graph2) select a.*,b.* from a,b where a.dst=b.src", dialect, ); - println!("AST {:?}", plan); + info!("AST {:?}", plan); let df = ctx .sql("WITH e AS (SELECT * FROM graph) SELECT e.* FROM e") diff --git a/raphtory-cypher/src/hop/execution.rs b/raphtory-cypher/src/hop/execution.rs index 7349d99d47..55213c3b7a 100644 --- a/raphtory-cypher/src/hop/execution.rs +++ b/raphtory-cypher/src/hop/execution.rs @@ -353,7 +353,7 @@ fn produce_next_record( right_proj: Option>, ) -> Option> { // if let Some(VID(0)) = prev_node { - // println!( + // info!( // "produce_next_record row: {row_pos} edge: {edge_pos} layer: {layer_pos} time: {time_pos} max_rows: {max_record_rows}, prev_node: {prev_node:?}", // ); // } diff --git a/raphtory-cypher/src/hop/rule.rs b/raphtory-cypher/src/hop/rule.rs index 88d7019ff7..9a3a58e5fd 100644 --- a/raphtory-cypher/src/hop/rule.rs +++ b/raphtory-cypher/src/hop/rule.rs @@ -138,14 +138,15 @@ impl ExtensionPlanner for HopPlanner { #[cfg(test)] mod test { + use crate::prepare_plan; use arrow::util::pretty::print_batches; - use raphtory::disk_graph::DiskGraphStorage; + use raphtory::{disk_graph::DiskGraphStorage, logging::global_info_logger}; use tempfile::tempdir; - - use crate::prepare_plan; + use tracing::info; #[tokio::test] async fn double_hop_edge_to_edge() { + global_info_logger(); let graph_dir = tempdir().unwrap(); let edges = vec![(0u64, 1u64, 0i64, 2.)]; let g = DiskGraphStorage::make_simple_graph(graph_dir, &edges, 10, 10); @@ -153,11 +154,12 @@ mod test { .await .unwrap(); - println!("PLAN {plan:?}"); + info!("PLAN {plan:?}"); } #[tokio::test] async fn double_hop_edge_to_edge_with_pushdown_filter_e2() { + global_info_logger(); let graph_dir = tempdir().unwrap(); let edges = vec![(0u64, 1u64, 0i64, 2.)]; let g = DiskGraphStorage::make_simple_graph(graph_dir, &edges, 10, 10); @@ -169,11 +171,12 @@ mod test { .await .unwrap(); - println!("PLAN {plan:?}"); + info!("PLAN {plan:?}"); } #[tokio::test] async fn double_hop_edge_to_edge_with_pushdown_filter_e1() { + global_info_logger(); let graph_dir = tempdir().unwrap(); let edges = vec![(0u64, 1u64, 0i64, 2.)]; let g = DiskGraphStorage::make_simple_graph(graph_dir, &edges, 10, 10); @@ -185,11 +188,12 @@ mod test { .await .unwrap(); - println!("PLAN {plan:?}"); + info!("PLAN {plan:?}"); } #[tokio::test] async fn as_physical_plan_e1() { + global_info_logger(); // +----+----------+-----+-----+----------+--------+----+----------+-----+-----+----------+--------+ // | id | layer_id | src | dst | rap_time | weight | id | layer_id | src | dst | rap_time | weight | // +----+----------+-----+-----+----------+--------+----+----------+-----+-----+----------+--------+ @@ -205,7 +209,7 @@ mod test { .await .unwrap(); - println!("PLAN {plan:?}"); + info!("PLAN {plan:?}"); let df = ctx.execute_logical_plan(plan).await.unwrap(); let out = df.collect().await.unwrap(); print_batches(&out).expect("print_batches"); diff --git a/raphtory-cypher/src/parser/mod.rs b/raphtory-cypher/src/parser/mod.rs index cb32c8ac55..dce824504b 100644 --- a/raphtory-cypher/src/parser/mod.rs +++ b/raphtory-cypher/src/parser/mod.rs @@ -1198,14 +1198,15 @@ mod test { #[test] fn and_expression() { + global_info_logger(); let input = "a.name = 'John' and 1 < a.age"; let pairs = CypherParser::parse(Rule::Expression, input); assert!(pairs.is_ok()); - println!("{:?}", pairs); + info!("{:?}", pairs); let expr = parse_expr(pairs.unwrap()); - println!("{:?}", expr); + info!("{:?}", expr); assert_eq!( expr, Ok(Expr::and( @@ -1297,6 +1298,8 @@ mod test { } use pretty_assertions::assert_eq; + use raphtory::logging::global_info_logger; + use tracing::info; #[test] fn parse_lanl_large_paths_with_expr() { diff --git a/raphtory-graphql/Cargo.toml b/raphtory-graphql/Cargo.toml index 60e22a465e..65bbc828ab 100644 --- a/raphtory-graphql/Cargo.toml +++ b/raphtory-graphql/Cargo.toml @@ -32,7 +32,7 @@ futures-util = { workspace = true } jsonwebtoken = { workspace = true } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } -opentelemetry-jaeger = { workspace = true } +opentelemetry-otlp = { workspace = true } tracing = { workspace = true } tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true } @@ -49,7 +49,6 @@ moka = { workspace = true } pyo3 = { workspace = true, optional = true } crossbeam-channel = { workspace = true } minijinja = { workspace = true } - [dev-dependencies] tempfile = { workspace = true } diff --git a/raphtory-graphql/src/azure_auth/common.rs b/raphtory-graphql/src/azure_auth/common.rs deleted file mode 100644 index 6c2680c083..0000000000 --- a/raphtory-graphql/src/azure_auth/common.rs +++ /dev/null @@ -1,245 +0,0 @@ -use base64_compat::{decode_config, URL_SAFE_NO_PAD}; -use chrono::{Duration, Utc}; -use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, Validation}; -use oauth2::{ - basic::BasicClient, reqwest::async_http_client, AuthorizationCode, CsrfToken, - PkceCodeChallenge, PkceCodeVerifier, Scope, TokenResponse, -}; -use poem::{ - handler, - http::StatusCode, - web::{ - cookie::{Cookie, CookieJar}, - Data, Json, Query, Redirect, - }, - IntoResponse, Response, -}; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use std::{ - collections::HashMap, - env, - error::Error, - sync::{Arc, Mutex}, -}; - -#[derive(Deserialize, Serialize)] -struct AuthRequest { - code: String, - state: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Jwks { - pub(crate) keys: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Jwk { - pub(crate) kid: String, - kty: String, - #[serde(rename = "use")] - use_: String, - pub(crate) n: String, - pub(crate) e: String, -} - -#[derive(Clone)] -pub struct AppState { - pub(crate) oauth_client: Arc, - pub(crate) csrf_state: Arc>>, - pub(crate) pkce_verifier: Arc>>, - pub(crate) jwks: Arc, -} - -#[handler] -pub async fn login(data: Data<&AppState>, jar: &CookieJar) -> Redirect { - let session_id = uuid::Uuid::new_v4().to_string(); - let (pkce_challenge, pkce_verifier) = PkceCodeChallenge::new_random_sha256(); - let client_id_str = env::var("CLIENT_ID").expect("CLIENT_ID not set"); - let (authorize_url, csrf_state) = data - .oauth_client - .authorize_url(CsrfToken::new_random) - .set_pkce_challenge(pkce_challenge) - .add_scope(Scope::new("openid".to_string())) - .add_scope(Scope::new("email".to_string())) - .add_scope(Scope::new("offline_access".to_string())) - .add_scope(Scope::new( - format!("{}/public-scope", client_id_str).to_string(), - )) - .url(); - - data.csrf_state - .lock() - .unwrap() - .insert(session_id.clone(), csrf_state); - data.pkce_verifier - .lock() - .unwrap() - .insert(session_id.clone(), pkce_verifier); - - let mut session_cookie = Cookie::new("session_id", session_id); - session_cookie.set_path("/"); - session_cookie.set_http_only(true); - jar.add(session_cookie); - - Redirect::temporary(authorize_url.to_string().as_str()) -} - -#[handler] -pub async fn auth_callback( - data: Data<&AppState>, - query: Query, - jar: &CookieJar, -) -> impl IntoResponse { - if let Some(session_cookie) = jar.get("session_id") { - let session_id = session_cookie.value::().unwrap(); - - let code = AuthorizationCode::new(query.0.code.clone()); - let pkce_verifier = data - .pkce_verifier - .lock() - .unwrap() - .remove(&session_id) - .unwrap(); - - let token_result = data - .oauth_client - .exchange_code(code) - .set_pkce_verifier(pkce_verifier) - .request_async(async_http_client) - .await; - - match token_result { - Ok(token) => { - let access_token = token.access_token(); - let expires_in = token - .expires_in() - .unwrap_or(core::time::Duration::from_secs(60 * 60 * 24)); - let expiration = Utc::now() + Duration::from_std(expires_in).unwrap(); - - let token_data = json!({ - "access_token_secret": access_token.secret(), - "expires_at": expiration.to_rfc3339() - }); - - let mut auth_cookie = Cookie::new("auth_token", token_data.to_string()); - auth_cookie.set_expires(expiration); - auth_cookie.set_path("/"); - auth_cookie.set_http_only(true); - jar.add(auth_cookie); - return Redirect::temporary("/").into_response(); - } - Err(_err) => { - return Response::builder() - .status(StatusCode::UNAUTHORIZED) - .content_type("application/json") - .body(json!({"error": "Login failed"}).to_string()); - } - } - } else { - return Response::builder() - .status(StatusCode::UNAUTHORIZED) - .content_type("application/json") - .body(json!({"error": "Session not found. Please login again"}).to_string()); - } -} - -pub fn decode_base64_urlsafe(base64_str: &str) -> Result, Box> { - let decoded = decode_config(base64_str, URL_SAFE_NO_PAD)?; - Ok(decoded) -} - -#[handler] -pub async fn secure_endpoint() -> Json { - Json(serde_json::json!({ - "message": "Secured" - })) -} - -#[handler] -pub async fn logout(jar: &CookieJar) -> String { - if let Some(mut cookie) = jar.get("auth_token") { - cookie.set_expires(Utc::now() - Duration::days(1)); - jar.remove("auth_token"); - } - if let Some(mut cookie) = jar.get("session_id") { - cookie.set_expires(Utc::now() - Duration::days(1)); - jar.remove("session_id"); - } - "You have been logged out.".to_string() -} - -#[handler] -pub async fn verify(data: Data<&AppState>, jar: &CookieJar) -> Json { - if let Some(_session_cookie) = jar.get("session_id") { - if let Some(cookie) = jar.get("auth_token") { - let cookie_value = cookie.value::().expect("Unable to find cookie"); - let token_data: serde_json::Value = - serde_json::from_str(&cookie_value).expect("Invalid cookie format"); - let token = token_data["access_token_secret"] - .as_str() - .expect("No access token found"); - let expires_at_str = token_data["expires_at"] - .as_str() - .expect("No expiration time found"); - - let expires_at = chrono::DateTime::parse_from_rfc3339(expires_at_str) - .expect("Invalid expiration format"); - - if Utc::now() > expires_at { - return Json(serde_json::json!({ - "message": "Access token expired, please login again" - })); - } - - let header = decode_header(token).expect("Unable to decode header"); - let kid = header.kid.expect("Token header does not have a kid field"); - - let jwk = data - .jwks - .keys - .iter() - .find(|&jwk| jwk.kid == kid) - .expect("Key ID not found in JWKS"); - - let n = decode_base64_urlsafe(&jwk.n).unwrap(); - let e = decode_base64_urlsafe(&jwk.e).unwrap(); - - let decoding_key = DecodingKey::from_rsa_raw_components(&n, &e); - - let validation = Validation::new(Algorithm::RS256); - - let token_data = - decode::>(token, &decoding_key, &validation); - - match token_data { - Ok(_dc) => Json(serde_json::json!({ - "message": "Valid access token", - })), - Err(_err) => Json(serde_json::json!({ - "message": "No valid auth token found", - })), - } - } else { - Json(serde_json::json!({ - "message": "No cookie auth_token found, please login" - })) - } - } else { - Json(serde_json::json!({ - "message": "No session_id found, please login" - })) - } -} - -#[allow(dead_code)] -pub async fn get_jwks() -> Result> { - let authority = env::var("AUTHORITY").expect("AUTHORITY not set"); - let jwks_url = format!("{}/discovery/v2.0/keys", authority); - let client = Client::new(); - let response = client.get(&jwks_url).send().await?; - let jwks = response.json::().await?; - Ok(jwks) -} diff --git a/raphtory-graphql/src/azure_auth/mod.rs b/raphtory-graphql/src/azure_auth/mod.rs deleted file mode 100644 index 423dcd51fb..0000000000 --- a/raphtory-graphql/src/azure_auth/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod common; - -pub(crate) mod token_middleware; diff --git a/raphtory-graphql/src/azure_auth/token_middleware.rs b/raphtory-graphql/src/azure_auth/token_middleware.rs deleted file mode 100644 index 4335cd01a4..0000000000 --- a/raphtory-graphql/src/azure_auth/token_middleware.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::azure_auth::common::{decode_base64_urlsafe, AppState}; -use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, Validation}; -use poem::{ - http::StatusCode, web::Redirect, Endpoint, Error, IntoResponse, Middleware, Request, Response, - Result, -}; -use std::{collections::HashMap, sync::Arc}; - -#[derive(Clone)] -pub struct TokenMiddleware { - app_state: Arc, -} - -impl TokenMiddleware { - #[allow(dead_code)] - pub fn new(app_state: Arc) -> Self { - TokenMiddleware { app_state } - } -} - -impl Middleware for TokenMiddleware { - type Output = TokenMiddlewareImpl; - - fn transform(&self, ep: E) -> Self::Output { - TokenMiddlewareImpl { - ep, - app_state: self.app_state.clone(), - } - } -} - -pub struct TokenMiddlewareImpl { - ep: E, - app_state: Arc, -} - -#[allow(dead_code)] -#[derive(Clone)] -struct Token(String); - -impl Endpoint for TokenMiddlewareImpl { - type Output = Response; - - async fn call(&self, mut req: Request) -> Result { - let jar = req.cookie().clone(); - if let Some(_session_cookie) = jar.get("session_id") { - if let Some(auth_cookie) = jar.get("auth_token") { - let token_data: serde_json::Value = serde_json::from_str( - &auth_cookie - .value::() - .expect("Unable to find cookie"), - ) - .map_err(|_| Error::from_status(StatusCode::UNAUTHORIZED))?; - let access_token = token_data["access_token_secret"] - .as_str() - .ok_or_else(|| Error::from_status(StatusCode::UNAUTHORIZED))?; - let expires_at_str = token_data["expires_at"] - .as_str() - .ok_or_else(|| Error::from_status(StatusCode::UNAUTHORIZED))?; - let expires_at = chrono::DateTime::parse_from_rfc3339(expires_at_str) - .map_err(|_| Error::from_status(StatusCode::UNAUTHORIZED))?; - if chrono::Utc::now() > expires_at { - return Err(Error::from_status(StatusCode::UNAUTHORIZED)); - } - - let header = decode_header(access_token) - .map_err(|_| Error::from_status(StatusCode::UNAUTHORIZED))?; - let kid = header - .kid - .ok_or_else(|| Error::from_status(StatusCode::UNAUTHORIZED))?; - - let jwk = self - .app_state - .jwks - .keys - .iter() - .find(|&jwk| jwk.kid == kid) - .ok_or_else(|| Error::from_status(StatusCode::UNAUTHORIZED))?; - - let n = decode_base64_urlsafe(&jwk.n) - .map_err(|_| Error::from_status(StatusCode::UNAUTHORIZED))?; - let e = decode_base64_urlsafe(&jwk.e) - .map_err(|_| Error::from_status(StatusCode::UNAUTHORIZED))?; - - let decoding_key = DecodingKey::from_rsa_raw_components(&n, &e); - - let validation = Validation::new(Algorithm::RS256); - decode::>( - access_token, - &decoding_key, - &validation, - ) - .map_err(|_| Error::from_status(StatusCode::UNAUTHORIZED))?; - - req.extensions_mut().insert(Token(access_token.to_string())); - - return self.ep.call(req).await.map(IntoResponse::into_response); - } - Ok(Redirect::temporary("/login").into_response()) - } else { - Ok(Redirect::temporary("/login").into_response()) - } - } -} diff --git a/raphtory-graphql/src/config/app_config.rs b/raphtory-graphql/src/config/app_config.rs new file mode 100644 index 0000000000..8244c5857e --- /dev/null +++ b/raphtory-graphql/src/config/app_config.rs @@ -0,0 +1,144 @@ +use crate::config::{ + cache_config::CacheConfig, log_config::LoggingConfig, otlp_config::TracingConfig, +}; +use config::{Config, ConfigError, File}; +use serde::Deserialize; +use std::path::PathBuf; + +#[derive(Debug, Deserialize, PartialEq, Clone)] +pub struct AppConfig { + pub logging: LoggingConfig, + pub cache: CacheConfig, + pub tracing: TracingConfig, +} + +impl Default for AppConfig { + fn default() -> Self { + Self { + logging: LoggingConfig::default(), + cache: CacheConfig::default(), + tracing: TracingConfig::default(), + } + } +} + +pub struct AppConfigBuilder { + logging: LoggingConfig, + cache: CacheConfig, + tracing: TracingConfig, +} + +impl AppConfigBuilder { + pub fn new() -> Self { + Self { + logging: LoggingConfig::default(), + cache: CacheConfig::default(), + tracing: TracingConfig::default(), + } + } + + pub fn from(config: AppConfig) -> Self { + Self { + logging: config.logging, + cache: config.cache, + tracing: config.tracing, + } + } + + pub fn with_log_level(mut self, log_level: String) -> Self { + self.logging.log_level = log_level; + self + } + + pub fn with_tracing(mut self, tracing: bool) -> Self { + self.tracing.tracing_enabled = tracing; + self + } + + pub fn with_otlp_agent_host(mut self, otlp_agent_host: String) -> Self { + self.tracing.otlp_agent_host = otlp_agent_host; + self + } + + pub fn with_otlp_agent_port(mut self, otlp_agent_port: String) -> Self { + self.tracing.otlp_agent_port = otlp_agent_port; + self + } + + pub fn with_otlp_tracing_service_name(mut self, otlp_tracing_service_name: String) -> Self { + self.tracing.otlp_tracing_service_name = otlp_tracing_service_name; + self + } + + pub fn with_cache_capacity(mut self, cache_capacity: u64) -> Self { + self.cache.capacity = cache_capacity; + self + } + + pub fn with_cache_tti_seconds(mut self, tti_seconds: u64) -> Self { + self.cache.tti_seconds = tti_seconds; + self + } + + pub fn build(self) -> AppConfig { + AppConfig { + logging: self.logging, + cache: self.cache, + tracing: self.tracing, + } + } +} + +// Order of precedence of config loading: config args >> config path >> config default +// Note: Since config args takes precedence over config path, ensure not to provide config args when starting server from a compile rust instance. +// This would cause configs from config paths to be ignored. The reason it has been implemented so is to avoid having to pass all the configs as +// args from the python instance i.e., being able to provide configs from config path as default configs and yet give precedence to config args. +pub fn load_config( + app_config: Option, + config_path: Option, +) -> Result { + let mut settings_config_builder = Config::builder(); + if let Some(config_path) = config_path { + settings_config_builder = settings_config_builder.add_source(File::from(config_path)); + } + let settings = settings_config_builder.build()?; + + let mut app_config_builder = if let Some(app_config) = app_config { + AppConfigBuilder::from(app_config) + } else { + AppConfigBuilder::new() + }; + + // Override with provided configs from config file if any + if let Some(log_level) = settings.get::("logging.log_level").ok() { + app_config_builder = app_config_builder.with_log_level(log_level); + } + if let Some(tracing) = settings.get::("tracing.tracing_enabled").ok() { + app_config_builder = app_config_builder.with_tracing(tracing); + } + + if let Some(otlp_agent_host) = settings.get::("tracing.otlp_agent_host").ok() { + app_config_builder = app_config_builder.with_otlp_agent_host(otlp_agent_host); + } + + if let Some(otlp_agent_port) = settings.get::("tracing.otlp_agent_port").ok() { + app_config_builder = app_config_builder.with_otlp_agent_port(otlp_agent_port); + } + + if let Some(otlp_tracing_service_name) = settings + .get::("tracing.otlp_tracing_service_name") + .ok() + { + app_config_builder = + app_config_builder.with_otlp_tracing_service_name(otlp_tracing_service_name); + } + + if let Some(cache_capacity) = settings.get::("cache.capacity").ok() { + app_config_builder = app_config_builder.with_cache_capacity(cache_capacity); + } + if let Some(cache_tti_seconds) = settings.get::("cache.tti_seconds").ok() { + app_config_builder = app_config_builder.with_cache_tti_seconds(cache_tti_seconds); + } + + Ok(app_config_builder.build()) +} diff --git a/raphtory-graphql/src/config/cache_config.rs b/raphtory-graphql/src/config/cache_config.rs new file mode 100644 index 0000000000..eeb01afc06 --- /dev/null +++ b/raphtory-graphql/src/config/cache_config.rs @@ -0,0 +1,15 @@ +use serde::Deserialize; +#[derive(Debug, Deserialize, PartialEq, Clone)] +pub struct CacheConfig { + pub capacity: u64, + pub tti_seconds: u64, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { + capacity: 30, + tti_seconds: 900, + } + } +} diff --git a/raphtory-graphql/src/config/log_config.rs b/raphtory-graphql/src/config/log_config.rs new file mode 100644 index 0000000000..1dec9139b2 --- /dev/null +++ b/raphtory-graphql/src/config/log_config.rs @@ -0,0 +1,21 @@ +use serde::Deserialize; +use tracing_subscriber::EnvFilter; + +#[derive(Debug, Deserialize, PartialEq, Clone)] +pub struct LoggingConfig { + pub log_level: String, +} + +impl LoggingConfig { + pub fn get_log_env(&self) -> EnvFilter { + raphtory_api::core::utils::logging::get_log_env(self.log_level.clone()) + } +} + +impl Default for LoggingConfig { + fn default() -> Self { + Self { + log_level: "INFO".to_string(), + } + } +} diff --git a/raphtory-graphql/src/config/mod.rs b/raphtory-graphql/src/config/mod.rs new file mode 100644 index 0000000000..2efe8be4bf --- /dev/null +++ b/raphtory-graphql/src/config/mod.rs @@ -0,0 +1,51 @@ +pub mod app_config; +pub mod cache_config; +pub mod log_config; +pub mod otlp_config; + +#[cfg(test)] +mod tests { + use crate::config::app_config::{load_config, AppConfigBuilder}; + use std::{fs, path::PathBuf}; + + #[test] + fn test_load_config_from_toml() { + let config_toml = r#" + [logging] + log_level = "DEBUG" + + [tracing] + tracing_enabled = true + + [cache] + tti_seconds = 1000 + "#; + let config_path = PathBuf::from("test_config.toml"); + fs::write(&config_path, config_toml).unwrap(); + + let result = load_config(None, Some(config_path.clone())); + let expected_config = AppConfigBuilder::new() + .with_log_level("DEBUG".to_string()) + .with_tracing(true) + .with_cache_capacity(30) + .with_cache_tti_seconds(1000) + .build(); + + assert_eq!(result.unwrap(), expected_config); + + // Cleanup: delete the test TOML file + fs::remove_file(config_path).unwrap(); + } + + #[test] + fn test_load_config_with_custom_cache() { + let app_config = AppConfigBuilder::new() + .with_cache_capacity(50) + .with_cache_tti_seconds(1200) + .build(); + + let result = load_config(Some(app_config.clone()), None); + + assert_eq!(result.unwrap(), app_config); + } +} diff --git a/raphtory-graphql/src/config/otlp_config.rs b/raphtory-graphql/src/config/otlp_config.rs new file mode 100644 index 0000000000..22af1d81fd --- /dev/null +++ b/raphtory-graphql/src/config/otlp_config.rs @@ -0,0 +1,71 @@ +use opentelemetry::KeyValue; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::{ + trace, + trace::{Sampler, TracerProvider}, + Resource, +}; +use serde::Deserialize; +use std::time::Duration; +use tracing::{error, info}; + +#[derive(Clone, Deserialize, Debug, PartialEq)] +pub struct TracingConfig { + pub tracing_enabled: bool, + pub otlp_agent_host: String, + pub otlp_agent_port: String, + pub otlp_tracing_service_name: String, +} +impl Default for TracingConfig { + fn default() -> Self { + Self { + tracing_enabled: false, + otlp_agent_host: "http://localhost".to_string(), + otlp_agent_port: "4317".to_string(), + otlp_tracing_service_name: "Raphtory".to_string(), + } + } +} +impl TracingConfig { + pub fn tracer_provider(&self) -> Option { + if self.tracing_enabled { + match opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint(format!( + "{}:{}", + self.otlp_agent_host.clone(), + self.otlp_agent_port.clone() + )) + .with_timeout(Duration::from_secs(3)), + ) + .with_trace_config( + trace::Config::default() + .with_sampler(Sampler::AlwaysOn) + .with_resource(Resource::new(vec![KeyValue::new( + "service.name", + self.otlp_tracing_service_name.clone(), + )])), + ) + .install_batch(opentelemetry_sdk::runtime::Tokio) + { + Ok(tracer_provider) => { + info!( + "Sending traces to {}:{}", + self.otlp_agent_host.clone(), + self.otlp_agent_port.clone() + ); + Some(tracer_provider) + } + Err(e) => { + error!("{}", e.to_string()); + None + } + } + } else { + None + } + } +} diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index d926d9ff75..85108aef35 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -1,6 +1,6 @@ use crate::{ + config::app_config::AppConfig, model::{algorithms::global_plugins::GlobalPlugins, create_dirs_if_not_present, GqlGraphType}, - server_config::AppConfig, }; use moka::sync::Cache; #[cfg(feature = "storage")] @@ -26,6 +26,7 @@ use std::{ path::{Component, Path, PathBuf, StripPrefixError}, sync::Arc, }; +use tracing::{error, info}; use walkdir::WalkDir; pub struct Data { @@ -44,7 +45,7 @@ impl Data { .eviction_listener(|_, value, _| { value .write_updates() - .unwrap_or_else(|err| println!("Write on eviction failed: {err:?}")) + .unwrap_or_else(|err| error!("Write on eviction failed: {err:?}")) }) .build(); @@ -203,7 +204,7 @@ impl Data { .map(move |entry| { let path = entry.path(); let path_string = path.display().to_string(); - println!("loading from {path_string}"); + info!("loading from {path_string}"); loader(path) }) } @@ -241,13 +242,13 @@ fn load_disk_graph_from_path( if overwrite { fs::remove_dir_all(&target_path)?; copy_dir_recursive(path_on_server, &target_path)?; - println!("Disk Graph loaded = {}", target_path.display()); + info!("Disk Graph loaded = {}", target_path.display()); } else { return Err(GraphError::GraphNameAlreadyExists(target_path.to_path_buf()).into()); } } else { copy_dir_recursive(path_on_server, &target_path)?; - println!("Disk Graph loaded = {}", target_path.display()); + info!("Disk Graph loaded = {}", target_path.display()); } Ok(Some(target_path.to_path_buf())) } @@ -257,7 +258,7 @@ fn get_disk_graph_from_path( path: &Path, ) -> Result>, GraphError> { let graph = load_disk_graph(path)?; - println!("Disk Graph loaded = {}", path.display()); + info!("Disk Graph loaded = {}", path.display()); Ok(Some(IndexedGraph::from_graph(&graph.into())?)) } @@ -280,7 +281,7 @@ fn get_graph_from_path(path: &Path) -> Result, G } } else { let graph = MaterializedGraph::load_cached(path)?; - println!("Graph loaded = {}", path.display()); + info!("Graph loaded = {}", path.display()); Ok(IndexedGraph::from_graph(&graph)?) } } @@ -342,10 +343,7 @@ fn _load_disk_graph(_path: &Path) -> Result { #[cfg(test)] pub(crate) mod data_tests { - use crate::{ - data::{get_graph_from_path, Data}, - server_config::{AppConfig, AppConfigBuilder}, - }; + use crate::data::{get_graph_from_path, Data}; use raphtory::{db::api::view::MaterializedGraph, prelude::*}; use std::{ collections::HashMap, @@ -355,6 +353,7 @@ pub(crate) mod data_tests { path::{Path, PathBuf}, }; + use crate::config::app_config::{AppConfig, AppConfigBuilder}; #[cfg(feature = "storage")] use crate::data::copy_dir_recursive; use raphtory::core::utils::errors::{GraphError, InvalidPathReason}; diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 32afcd6ba7..7d8f442fb1 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -1,22 +1,21 @@ pub use crate::server::GraphServer; -pub mod azure_auth; mod data; pub mod model; -mod observability; +pub mod observability; mod routes; pub mod server; -pub mod server_config; pub mod url_encode; +pub mod config; #[cfg(feature = "python")] pub mod python; #[cfg(test)] mod graphql_test { use crate::{ + config::app_config::AppConfig, data::{data_tests::save_graphs_to_work_dir, Data}, model::App, - server_config::AppConfig, url_encode::{url_decode_graph, url_encode_graph}, }; use async_graphql::UploadValue; diff --git a/raphtory-graphql/src/main.rs b/raphtory-graphql/src/main.rs index db2fa78ab1..9be9ca1d6c 100644 --- a/raphtory-graphql/src/main.rs +++ b/raphtory-graphql/src/main.rs @@ -1,4 +1,4 @@ -use raphtory_graphql::GraphServer; +use raphtory_graphql::{config::app_config::AppConfigBuilder, GraphServer}; use std::{ env, path::{Path, PathBuf}, @@ -10,8 +10,8 @@ async fn main() -> IoResult<()> { let default_path = Path::new("/tmp/graphs"); let work_dir = env::var("GRAPH_DIRECTORY").unwrap_or(default_path.display().to_string()); let work_dir = PathBuf::from(&work_dir); - - GraphServer::new(work_dir, None, None)?.run().await?; + let app_config = Some(AppConfigBuilder::new().with_tracing(true).build()); + GraphServer::new(work_dir, app_config, None)?.run().await?; Ok(()) } diff --git a/raphtory-graphql/src/model/algorithms/global_search.rs b/raphtory-graphql/src/model/algorithms/global_search.rs index 9a36a4898e..13cdbb4f9f 100644 --- a/raphtory-graphql/src/model/algorithms/global_search.rs +++ b/raphtory-graphql/src/model/algorithms/global_search.rs @@ -9,6 +9,7 @@ use dynamic_graphql::internal::TypeName; use futures_util::future::BoxFuture; use raphtory::vectors::{embeddings::openai_embedding, vectorised_cluster::VectorisedCluster}; use std::ops::Deref; +use tracing::info; pub(crate) struct GlobalSearch; @@ -39,7 +40,7 @@ impl<'a> Algorithm<'a, GlobalPlugins> for GlobalSearch { Box::pin(async move { let embedding = openai_embedding(vec![query.clone()]).await.remove(0); - println!("running global search for {query}"); + info!("running global search for {query}"); let graphs = vectorised_graphs.read(); diff --git a/raphtory-graphql/src/model/algorithms/similarity_search.rs b/raphtory-graphql/src/model/algorithms/similarity_search.rs index 2d99a15b0c..64df114e7e 100644 --- a/raphtory-graphql/src/model/algorithms/similarity_search.rs +++ b/raphtory-graphql/src/model/algorithms/similarity_search.rs @@ -8,6 +8,7 @@ use async_graphql::{ use dynamic_graphql::internal::TypeName; use futures_util::future::BoxFuture; use raphtory::vectors::embeddings::openai_embedding; +use tracing::info; pub(crate) struct SimilaritySearch; @@ -38,7 +39,7 @@ impl<'a> Algorithm<'a, VectorAlgorithms> for SimilaritySearch { Box::pin(async move { let embedding = openai_embedding(vec![query.clone()]).await.remove(0); - println!("running similarity search for {query}"); + info!("running similarity search for {query}"); let documents = graph .documents_by_similarity(&embedding, limit, None) diff --git a/raphtory-graphql/src/model/graph/graph.rs b/raphtory-graphql/src/model/graph/graph.rs index ef146abdda..a18a4521d0 100644 --- a/raphtory-graphql/src/model/graph/graph.rs +++ b/raphtory-graphql/src/model/graph/graph.rs @@ -26,14 +26,7 @@ use raphtory::{ prelude::*, search::{into_indexed::DynamicIndexedGraph, IndexedGraph}, }; -use std::{ - collections::HashSet, - convert::Into, - fs, io, - path::{Path, PathBuf}, - sync::Arc, - time::UNIX_EPOCH, -}; +use std::{collections::HashSet, convert::Into, fs, path::PathBuf, sync::Arc, time::UNIX_EPOCH}; #[derive(ResolvedObject)] pub(crate) struct GqlGraph { @@ -146,6 +139,7 @@ impl GqlGraph { } /// Return a graph containing only the activity between `start` and `end` measured as milliseconds from epoch + async fn window(&self, start: i64, end: i64) -> GqlGraph { GqlGraph::new( self.work_dir.clone(), @@ -308,6 +302,7 @@ impl GqlGraph { async fn count_nodes(&self) -> usize { self.graph.count_nodes() } + async fn search_node_count(&self, query: String) -> usize { self.graph.search_node_count(&query).unwrap_or(0) } @@ -319,9 +314,11 @@ impl GqlGraph { async fn has_node(&self, name: String) -> bool { self.graph.has_node(name) } + async fn has_node_id(&self, id: u64) -> bool { self.graph.has_node(id) } + async fn has_edge(&self, src: String, dst: String, layer: Option) -> bool { match layer { Some(name) => self @@ -436,9 +433,11 @@ impl GqlGraph { //These name/path functions basically can only fail //if someone write non-utf characters as a filename + async fn name(&self) -> Result { get_graph_name(&self.path) } + async fn path(&self) -> Result { self.path .to_str() diff --git a/raphtory-graphql/src/model/graph/graphs.rs b/raphtory-graphql/src/model/graph/graphs.rs index dacf2348bf..6942158ea2 100644 --- a/raphtory-graphql/src/model/graph/graphs.rs +++ b/raphtory-graphql/src/model/graph/graphs.rs @@ -20,6 +20,7 @@ impl GqlGraphs { impl GqlGraphs { //Name and path here do not return a result as we only want to let the user know about //valid graph paths. No point blowing up if there is one busted fule + async fn name(&self) -> Vec { self.paths .iter() diff --git a/raphtory-graphql/src/model/graph/mutable_graph.rs b/raphtory-graphql/src/model/graph/mutable_graph.rs index 2dbb17de42..8296db24cc 100644 --- a/raphtory-graphql/src/model/graph/mutable_graph.rs +++ b/raphtory-graphql/src/model/graph/mutable_graph.rs @@ -69,11 +69,13 @@ fn as_properties(properties: Vec) -> impl Iterator GqlGraph { GqlGraph::new(self.work_dir.clone(), self.path.clone(), self.graph.clone()) } /// Get mutable existing node + async fn node(&self, name: String) -> Option { self.graph.node(name).map(|n| n.into()) } @@ -130,6 +132,7 @@ impl GqlMutableGraph { } /// Get a mutable existing edge + async fn edge(&self, src: String, dst: String) -> Option { self.graph.edge(src, dst).map(|e| e.into()) } @@ -186,6 +189,7 @@ impl GqlMutableGraph { } /// Mark an edge as deleted (creates the edge if it did not exist) + async fn delete_edge( &self, time: i64, diff --git a/raphtory-graphql/src/model/graph/node.rs b/raphtory-graphql/src/model/graph/node.rs index c55a511dce..5281b68780 100644 --- a/raphtory-graphql/src/model/graph/node.rs +++ b/raphtory-graphql/src/model/graph/node.rs @@ -39,6 +39,7 @@ impl Node { //////////////////////// // LAYERS AND WINDOWS // //////////////////////// + async fn layers(&self, names: Vec) -> Node { self.vv.valid_layers(names).into() } @@ -86,6 +87,7 @@ impl Node { //////////////////////// //// TIME QUERIES ////// //////////////////////// + async fn earliest_time(&self) -> Option { self.vv.earliest_time() } @@ -117,6 +119,7 @@ impl Node { //////////////////////// /////// PROPERTIES ///// //////////////////////// + pub async fn node_type(&self) -> Option { match self.vv.node_type() { None => None, @@ -132,16 +135,19 @@ impl Node { //// EDGE GETTERS ////// //////////////////////// /// Returns the number of edges connected to this node + async fn degree(&self) -> usize { self.vv.degree() } /// Returns the number edges with this node as the source + async fn out_degree(&self) -> usize { self.vv.out_degree() } /// Returns the number edges with this node as the destination + async fn in_degree(&self) -> usize { self.vv.in_degree() } diff --git a/raphtory-graphql/src/model/graph/property.rs b/raphtory-graphql/src/model/graph/property.rs index 2fc3ee9253..bc196e6cea 100644 --- a/raphtory-graphql/src/model/graph/property.rs +++ b/raphtory-graphql/src/model/graph/property.rs @@ -99,6 +99,7 @@ impl GqlProp { async fn key(&self) -> String { self.key.clone() } + async fn as_string(&self) -> String { self.prop.to_string() } @@ -129,9 +130,11 @@ impl GqlPropTuple { async fn time(&self) -> i64 { self.time } + async fn as_string(&self) -> String { self.prop.to_string() } + async fn value(&self) -> GqlPropValue { GqlPropValue(self.prop.clone()) } @@ -158,18 +161,23 @@ impl GqlTemporalProp { async fn key(&self) -> String { self.key.clone() } + async fn history(&self) -> Vec { self.prop.history() } + async fn values(&self) -> Vec { self.prop.values().iter().map(|x| x.to_string()).collect() } + async fn at(&self, t: i64) -> Option { self.prop.at(t).map(|x| x.to_string()) } + async fn latest(&self) -> Option { self.prop.latest().map(|x| x.to_string()) } + async fn unique(&self) -> Vec { self.prop .unique() @@ -177,6 +185,7 @@ impl GqlTemporalProp { .map(|x| x.to_string()) .collect_vec() } + async fn ordered_dedupe(&self, latest_time: bool) -> Vec { self.prop .ordered_dedupe(latest_time) @@ -240,9 +249,11 @@ impl GqlProperties { async fn get(&self, key: &str) -> Option { self.props.get(key).map(|p| (key.to_string(), p).into()) } + async fn contains(&self, key: &str) -> bool { self.props.contains(key) } + async fn keys(&self) -> Vec { self.props.keys().map(|k| k.into()).collect() } @@ -273,7 +284,7 @@ impl GqlProperties { self.props.temporal().into() } - pub fn constant(&self) -> GqlConstantProperties { + async fn constant(&self) -> GqlConstantProperties { self.props.constant().into() } } @@ -283,9 +294,11 @@ impl GqlConstantProperties { async fn get(&self, key: &str) -> Option { self.props.get(key).map(|p| (key.to_string(), p).into()) } + async fn contains(&self, key: &str) -> bool { self.props.contains(key) } + async fn keys(&self) -> Vec { self.props.keys().iter().map(|k| k.clone().into()).collect() } @@ -318,12 +331,15 @@ impl GqlTemporalProperties { async fn get(&self, key: &str) -> Option { self.props.get(key).map(|p| (key.to_string(), p).into()) } + async fn contains(&self, key: &str) -> bool { self.props.contains(key) } + async fn keys(&self) -> Vec { self.props.keys().map(|k| k.into()).collect() } + async fn values(&self, keys: Option>) -> Vec { match keys { Some(keys) => self diff --git a/raphtory-graphql/src/observability/mod.rs b/raphtory-graphql/src/observability/mod.rs index 802a152647..a73bfdd7ee 100644 --- a/raphtory-graphql/src/observability/mod.rs +++ b/raphtory-graphql/src/observability/mod.rs @@ -1 +1 @@ -pub(crate) mod tracing; +pub mod open_telemetry; diff --git a/raphtory-graphql/src/observability/open_telemetry.rs b/raphtory-graphql/src/observability/open_telemetry.rs new file mode 100644 index 0000000000..240c142f5c --- /dev/null +++ b/raphtory-graphql/src/observability/open_telemetry.rs @@ -0,0 +1,203 @@ +use async_graphql::{ + async_trait, + extensions::{ + Extension, ExtensionContext, ExtensionFactory, NextExecute, NextParseQuery, NextRequest, + NextResolve, NextSubscribe, NextValidation, ResolveInfo, + }, + parser::types::ExecutableDocument, + QueryPathSegment, Response, ServerError, ServerResult, ValidationResult, Value, Variables, +}; +use futures_util::{stream::BoxStream, TryFutureExt}; +use opentelemetry::{ + trace::{FutureExt, SpanKind, TraceContextExt, Tracer}, + Context as OpenTelemetryContext, Key, +}; +use std::sync::Arc; + +const KEY_SOURCE: Key = Key::from_static_str("graphql.source"); +const KEY_VARIABLES: Key = Key::from_static_str("graphql.variables"); +const KEY_PARENT_TYPE: Key = Key::from_static_str("graphql.parentType"); +const KEY_RETURN_TYPE: Key = Key::from_static_str("graphql.returnType"); +const KEY_ERROR: Key = Key::from_static_str("graphql.error"); +const KEY_COMPLEXITY: Key = Key::from_static_str("graphql.complexity"); +const KEY_DEPTH: Key = Key::from_static_str("graphql.depth"); + +/// OpenTelemetry extension +#[cfg_attr(docsrs, doc(cfg(feature = "opentelemetry")))] +pub struct OpenTelemetry { + tracer: Arc, +} + +impl OpenTelemetry { + /// Use `tracer` to create an OpenTelemetry extension. + pub fn new(tracer: T) -> OpenTelemetry + where + T: Tracer + Send + Sync + 'static, + ::Span: Sync + Send, + { + Self { + tracer: Arc::new(tracer), + } + } +} + +impl ExtensionFactory for OpenTelemetry +where + T: Tracer + Send + Sync + 'static, + ::Span: Sync + Send, +{ + fn create(&self) -> Arc { + Arc::new(OpenTelemetryExtension { + tracer: self.tracer.clone(), + }) + } +} + +struct OpenTelemetryExtension { + tracer: Arc, +} + +#[async_trait::async_trait] +impl Extension for OpenTelemetryExtension +where + T: Tracer + Send + Sync + 'static, + ::Span: Sync + Send, +{ + async fn request(&self, ctx: &ExtensionContext<'_>, next: NextRequest<'_>) -> Response { + next.run(ctx) + .with_context(OpenTelemetryContext::current_with_span( + self.tracer + .span_builder("request") + .with_kind(SpanKind::Server) + .start(&*self.tracer), + )) + .await + } + + fn subscribe<'s>( + &self, + ctx: &ExtensionContext<'_>, + stream: BoxStream<'s, Response>, + next: NextSubscribe<'_>, + ) -> BoxStream<'s, Response> { + Box::pin( + next.run(ctx, stream) + .with_context(OpenTelemetryContext::current_with_span( + self.tracer + .span_builder("subscribe") + .with_kind(SpanKind::Server) + .start(&*self.tracer), + )), + ) + } + + async fn parse_query( + &self, + ctx: &ExtensionContext<'_>, + query: &str, + variables: &Variables, + next: NextParseQuery<'_>, + ) -> ServerResult { + let attributes = vec![ + KEY_SOURCE.string(query.to_string()), + KEY_VARIABLES.string(serde_json::to_string(variables).unwrap()), + ]; + let span = self + .tracer + .span_builder("parse") + .with_kind(SpanKind::Server) + .with_attributes(attributes) + .start(&*self.tracer); + + async move { + let res = next.run(ctx, query, variables).await; + if let Ok(doc) = &res { + OpenTelemetryContext::current() + .span() + .set_attribute(KEY_SOURCE.string(ctx.stringify_execute_doc(doc, variables))); + } + res + } + .with_context(OpenTelemetryContext::current_with_span(span)) + .await + } + + async fn validation( + &self, + ctx: &ExtensionContext<'_>, + next: NextValidation<'_>, + ) -> Result> { + let span = self + .tracer + .span_builder("validation") + .with_kind(SpanKind::Server) + .start(&*self.tracer); + next.run(ctx) + .with_context(OpenTelemetryContext::current_with_span(span)) + .map_ok(|res| { + let current_cx = OpenTelemetryContext::current(); + let span = current_cx.span(); + span.set_attribute(KEY_COMPLEXITY.i64(res.complexity as i64)); + span.set_attribute(KEY_DEPTH.i64(res.depth as i64)); + res + }) + .await + } + + async fn execute( + &self, + ctx: &ExtensionContext<'_>, + operation_name: Option<&str>, + next: NextExecute<'_>, + ) -> Response { + let span = self + .tracer + .span_builder("execute") + .with_kind(SpanKind::Server) + .start(&*self.tracer); + next.run(ctx, operation_name) + .with_context(OpenTelemetryContext::current_with_span(span)) + .await + } + + async fn resolve( + &self, + ctx: &ExtensionContext<'_>, + info: ResolveInfo<'_>, + next: NextResolve<'_>, + ) -> ServerResult> { + let span = if !info.is_for_introspection { + let attributes = vec![ + KEY_PARENT_TYPE.string(info.parent_type.to_string()), + KEY_RETURN_TYPE.string(info.return_type.to_string()), + ]; + match info.path_node.segment { + QueryPathSegment::Index(_) => None, + QueryPathSegment::Name(name) => Some( + self.tracer + .span_builder(name.to_string()) + .with_kind(SpanKind::Server) + .with_attributes(attributes) + .start(&*self.tracer), + ), + } + } else { + None + }; + + let fut = next.run(ctx, info).inspect_err(|err| { + let current_cx = OpenTelemetryContext::current(); + current_cx + .span() + .add_event("error".to_string(), vec![KEY_ERROR.string(err.to_string())]); + }); + + match span { + Some(span) => { + fut.with_context(OpenTelemetryContext::current_with_span(span)) + .await + } + None => fut.await, + } + } +} diff --git a/raphtory-graphql/src/observability/tracing.rs b/raphtory-graphql/src/observability/tracing.rs deleted file mode 100644 index 1e3753db67..0000000000 --- a/raphtory-graphql/src/observability/tracing.rs +++ /dev/null @@ -1,50 +0,0 @@ -use opentelemetry::global; -use opentelemetry_sdk::{ - propagation::TraceContextPropagator, - runtime::Tokio, - trace::{self, Sampler, Tracer}, -}; -use std::env; - -struct JaegerConfig { - jaeger_agent_host: String, - jaeger_agent_port: String, - jaeger_tracing_service_name: String, -} - -pub fn create_tracer_from_env() -> Option { - let jaeger_enabled: bool = env::var("JAEGER_ENABLED") - .unwrap_or_else(|_| "false".into()) - .parse() - .unwrap(); - - if jaeger_enabled { - let config = get_jaeger_config_from_env(); - Some(init_tracer(config)) - } else { - None - } -} - -fn init_tracer(config: JaegerConfig) -> Tracer { - global::set_text_map_propagator(TraceContextPropagator::new()); - opentelemetry_jaeger::new_agent_pipeline() - .with_endpoint(format!( - "{}:{}", - config.jaeger_agent_host, config.jaeger_agent_port - )) - .with_auto_split_batch(true) - .with_service_name(config.jaeger_tracing_service_name) - .with_trace_config(trace::config().with_sampler(Sampler::AlwaysOn)) - .install_batch(Tokio) - .expect("pipeline install error") -} - -fn get_jaeger_config_from_env() -> JaegerConfig { - JaegerConfig { - jaeger_agent_host: env::var("JAEGER_AGENT_HOST").unwrap_or_else(|_| "localhost".into()), - jaeger_agent_port: env::var("JAEGER_AGENT_PORT").unwrap_or_else(|_| "6831".into()), - jaeger_tracing_service_name: env::var("TRACING_SERVICE_NAME") - .unwrap_or_else(|_| "axum-graphql".into()), - } -} diff --git a/raphtory-graphql/src/python/client/mod.rs b/raphtory-graphql/src/python/client/mod.rs index 596826e1bf..7dd8b35fa5 100644 --- a/raphtory-graphql/src/python/client/mod.rs +++ b/raphtory-graphql/src/python/client/mod.rs @@ -270,9 +270,7 @@ fn to_graphql_valid(key: &String, value: &Prop) -> String { Prop::NDTime(value) => format!("{{ key: \"{}\", value: \"{}\" }}", key, value.to_string()), Prop::Graph(_) => "Graph cannot be converted to JSON".to_string(), Prop::PersistentGraph(_) => "Persistent Graph cannot be converted to JSON".to_string(), - Prop::Document(DocumentInput { content, .. }) => { - "Document cannot be converted to JSON".to_string() - } // TODO: return Value::Object ?? + Prop::Document(_) => "Document cannot be converted to JSON".to_string(), // TODO: return Value::Object ?? } } diff --git a/raphtory-graphql/src/python/client/raphtory_client.rs b/raphtory-graphql/src/python/client/raphtory_client.rs index 379393b4f0..6fc22f4f45 100644 --- a/raphtory-graphql/src/python/client/raphtory_client.rs +++ b/raphtory-graphql/src/python/client/raphtory_client.rs @@ -17,6 +17,7 @@ use reqwest::{multipart, multipart::Part, Client}; use serde_json::{json, Value as JsonValue}; use std::{collections::HashMap, fs::File, io::Read, path::Path}; use tokio::{self, runtime::Runtime}; +use tracing::debug; /// A client for handling GraphQL operations in the context of Raphtory. #[derive(Clone)] @@ -171,7 +172,7 @@ impl PyRaphtoryClient { match data.get("sendGraph") { Some(JsonValue::String(name)) => { - println!("Sent graph '{name}' to the server"); + debug!("Sent graph '{name}' to the server"); Ok(()) } _ => Err(PyException::new_err(format!( diff --git a/raphtory-graphql/src/python/server/running_server.rs b/raphtory-graphql/src/python/server/running_server.rs index 3e0b8dbfce..dc9b1e8c55 100644 --- a/raphtory-graphql/src/python/server/running_server.rs +++ b/raphtory-graphql/src/python/server/running_server.rs @@ -3,13 +3,14 @@ use crate::python::{ server::{is_online, wait_server, BridgeCommand}, RUNNING_SERVER_CONSUMED_MSG, WAIT_CHECK_INTERVAL_MILLIS, }; -use crossbeam_channel::Sender as CrossbeamSender; +use crossbeam_channel::{SendError, Sender as CrossbeamSender}; use pyo3::{exceptions::PyException, pyclass, pymethods, Py, PyObject, PyResult, Python}; use std::{ thread::{sleep, JoinHandle}, time::Duration, }; use tokio::{self, io::Result as IoResult}; +use tracing::error; /// A Raphtory server handler that also enables querying the server #[pyclass(name = "RunningGraphServer")] @@ -67,10 +68,12 @@ impl PyRunningGraphServer { pub(crate) fn stop_server(&mut self, py: Python) -> PyResult<()> { Self::apply_if_alive(self, |handler| { - handler - .sender - .send(BridgeCommand::StopServer) - .expect("Failed when sending cancellation signal"); + match handler.sender.send(BridgeCommand::StopServer) { + Ok(_) => {} + Err(e) => { + error!("Failed to establish Channel with server, this could be because you already have Raphtory running on this port. {}",e.to_string()) + } + } Ok(()) })?; let server = &mut self.server_handler; diff --git a/raphtory-graphql/src/python/server/server.rs b/raphtory-graphql/src/python/server/server.rs index d8181297ca..0341fec5e5 100644 --- a/raphtory-graphql/src/python/server/server.rs +++ b/raphtory-graphql/src/python/server/server.rs @@ -1,4 +1,5 @@ use crate::{ + config::app_config::AppConfigBuilder, model::algorithms::{ algorithm_entry_point::AlgorithmEntryPoint, document::GqlDocument, global_plugins::GlobalPlugins, vector_algorithms::VectorAlgorithms, @@ -10,7 +11,6 @@ use crate::{ running_server::PyRunningGraphServer, take_server_ownership, wait_server, BridgeCommand, }, }, - server_config::AppConfigBuilder, GraphServer, }; use async_graphql::dynamic::{Field, FieldFuture, FieldValue, InputValue, Object, TypeRef}; @@ -137,19 +137,36 @@ impl PyGraphServer { impl PyGraphServer { #[new] #[pyo3( - signature = (work_dir, cache_capacity = None, cache_tti_seconds = None, log_level = None, config_path = None) + signature = (work_dir, cache_capacity = None, cache_tti_seconds = None, log_level = None, tracing=None, otlp_agent_host=None, otlp_agent_port=None, otlp_tracing_service_name=None, config_path = None) )] fn py_new( work_dir: PathBuf, cache_capacity: Option, cache_tti_seconds: Option, log_level: Option, + tracing: Option, + otlp_agent_host: Option, + otlp_agent_port: Option, + otlp_tracing_service_name: Option, config_path: Option, ) -> PyResult { let mut app_config_builder = AppConfigBuilder::new(); if let Some(log_level) = log_level { app_config_builder = app_config_builder.with_log_level(log_level); } + if let Some(tracing) = tracing { + app_config_builder = app_config_builder.with_tracing(tracing); + } + if let Some(otlp_agent_host) = otlp_agent_host { + app_config_builder = app_config_builder.with_otlp_agent_host(otlp_agent_host); + } + if let Some(otlp_agent_port) = otlp_agent_port { + app_config_builder = app_config_builder.with_otlp_agent_port(otlp_agent_port); + } + if let Some(otlp_tracing_service_name) = otlp_tracing_service_name { + app_config_builder = + app_config_builder.with_otlp_tracing_service_name(otlp_tracing_service_name); + } if let Some(cache_capacity) = cache_capacity { app_config_builder = app_config_builder.with_cache_capacity(cache_capacity); } @@ -308,7 +325,7 @@ impl PyGraphServer { }); let mut server = PyRunningGraphServer::new(join_handle, sender, port)?; - if let Some(server_handler) = &server.server_handler { + if let Some(_server_handler) = &server.server_handler { let url = format!("http://localhost:{port}"); match PyRunningGraphServer::wait_for_server_online(&url, timeout_ms) { Ok(_) => return Ok(server), diff --git a/raphtory-graphql/src/server.rs b/raphtory-graphql/src/server.rs index c10b97b749..00889536ef 100644 --- a/raphtory-graphql/src/server.rs +++ b/raphtory-graphql/src/server.rs @@ -6,7 +6,6 @@ use crate::{ algorithms::{algorithm::Algorithm, algorithm_entry_point::AlgorithmEntryPoint}, App, }, - observability::tracing::create_tracer_from_env, routes::{graphql_playground, health}, }; use async_graphql_poem::GraphQL; @@ -22,8 +21,14 @@ use raphtory::{ vectors::{template::DocumentTemplate, vectorisable::Vectorisable, EmbeddingFunction}, }; -use crate::server_config::{load_config, AppConfig, LoggingConfig}; +use crate::{ + config::app_config::{load_config, AppConfig}, + observability::open_telemetry::OpenTelemetry, + server::ServerError::SchemaError, +}; use config::ConfigError; +use opentelemetry::trace::TracerProvider; +use opentelemetry_sdk::trace::{Tracer, TracerProvider as TP}; use std::{ fs, path::{Path, PathBuf}, @@ -37,10 +42,12 @@ use tokio::{ mpsc, mpsc::{Receiver, Sender}, }, + task, task::JoinHandle, }; +use tracing::{debug, error, info}; use tracing_subscriber::{ - layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, FmtSubscriber, Registry, + fmt, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt, Registry, }; use url::ParseError; @@ -60,6 +67,10 @@ pub enum ServerError { FailedToParseUrl(#[from] ParseError), #[error("Failed to fetch JWKS")] FailedToFetchJWKS, + #[error("Failed to load schema: {0}")] + SchemaError(String), + #[error("Failed to create endpoints: {0}")] + EndpointError(String), } impl From for io::Error { @@ -71,7 +82,7 @@ impl From for io::Error { /// A struct for defining and running a Raphtory GraphQL server pub struct GraphServer { data: Data, - configs: AppConfig, + config: AppConfig, } impl GraphServer { @@ -83,11 +94,10 @@ impl GraphServer { if !work_dir.exists() { fs::create_dir_all(&work_dir).unwrap(); } - let configs = + let config = load_config(app_config, config_path).map_err(|err| ServerError::ConfigError(err))?; - let data = Data::new(work_dir.as_path(), &configs); - - Ok(Self { data, configs }) + let data = Data::new(work_dir.as_path(), &config); + Ok(Self { data, config }) } /// Vectorise a subset of the graphs of the server. @@ -130,7 +140,7 @@ impl GraphServer { for graph_name in graph_names { let graph_cache = cache.join(&graph_name); let graph = graphs.read().get(&graph_name).unwrap().clone(); - println!("Loading embeddings for {graph_name} using cache from {graph_cache:?}"); + info!("Loading embeddings for {graph_name} using cache from {graph_cache:?}"); let vectorised = graph .into_dynamic() .vectorise( @@ -143,7 +153,7 @@ impl GraphServer { .await; stores.write().insert(graph_name, vectorised); } - println!("Embeddings were loaded successfully"); + info!("Embeddings were loaded successfully"); Ok(self) } @@ -167,41 +177,39 @@ impl GraphServer { /// Start the server on the port `port` and return a handle to it. pub async fn start_with_port(self, port: u16) -> IoResult { - fn configure_logger(configs: &LoggingConfig) { - let log_level = &configs.log_level; - let filter = EnvFilter::new(log_level); - let subscriber = FmtSubscriber::builder().with_env_filter(filter).finish(); - if let Err(err) = tracing::subscriber::set_global_default(subscriber) { - eprintln!( - "Log level cannot be updated within the same runtime environment: {}", - err - ); + let config = &self.config; + let filter = config.logging.get_log_env(); + let tracer_name = config.tracing.otlp_tracing_service_name.clone(); + let tp = config.tracing.tracer_provider(); + + // Create the base registry + let registry = Registry::default().with(filter).with( + fmt::layer().pretty().with_span_events(FmtSpan::NONE), //(FULL, NEW, ENTER, EXIT, CLOSE) + ); + match tp.clone() { + Some(tp) => { + registry + .with( + tracing_opentelemetry::layer().with_tracer(tp.tracer(tracer_name.clone())), + ) + .try_init() + .ok(); } - } - - configure_logger(&self.configs.logging); - - let registry = Registry::default().with(tracing_subscriber::fmt::layer().pretty()); - let env_filter = EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("INFO")); - - match create_tracer_from_env() { - Some(tracer) => registry - .with(tracing_opentelemetry::layer().with_tracer(tracer)) - .with(env_filter) - .try_init(), - None => registry.with(env_filter).try_init(), - } - .unwrap_or(()); + None => { + registry.try_init().ok(); + } + }; // it is important that this runs after algorithms have been pushed to PLUGIN_ALGOS static variable - - let app: CorsEndpoint> = self.generate_endpoint().await?; + let app: CorsEndpoint> = self + .generate_endpoint(tp.clone().map(|tp| tp.tracer(tracer_name))) + .await?; let (signal_sender, signal_receiver) = mpsc::channel(1); - println!("Playground: http://localhost:{port}"); + info!("Playground live at: http://0.0.0.0:{port}"); let server_task = Server::new(TcpListener::bind(format!("0.0.0.0:{port}"))) - .run_with_graceful_shutdown(app, server_termination(signal_receiver), None); + .run_with_graceful_shutdown(app, server_termination(signal_receiver, tp), None); let server_result = tokio::spawn(server_task); Ok(RunningGraphServer { @@ -210,10 +218,19 @@ impl GraphServer { }) } - async fn generate_endpoint(self) -> IoResult>> { + async fn generate_endpoint( + self, + tracer: Option, + ) -> Result>, ServerError> { let schema_builder = App::create_schema(); let schema_builder = schema_builder.data(self.data); - let schema = schema_builder.finish().unwrap(); + let schema = schema_builder; + let schema = if let Some(t) = tracer { + schema.extension(OpenTelemetry::new(t)).finish() + } else { + schema.finish() + } + .map_err(|e| SchemaError(e.to_string()))?; let app = Route::new() .at("/", get(graphql_playground).post(GraphQL::new(schema))) @@ -223,104 +240,6 @@ impl GraphServer { Ok(app) } - // async fn generate_microsoft_endpoint_with_auth( - // self, - // enable_tracing: bool, - // port: u16, - // ) -> IoResult>> { - // let schema_builder = App::create_schema(); - // let schema_builder = schema_builder.data(self.data); - // let schema = if enable_tracing { - // let schema_builder = schema_builder.extension(ApolloTracing); - // schema_builder.finish().unwrap() - // } else { - // schema_builder.finish().unwrap() - // }; - // - // dotenv().ok(); - // let client_id = self - // .configs - // .auth - // .client_id - // .ok_or(ServerError::MissingClientId)?; - // let client_secret = self - // .configs - // .auth - // .client_secret - // .ok_or(ServerError::MissingClientSecret)?; - // let tenant_id = self - // .configs - // .auth - // .tenant_id - // .ok_or(ServerError::MissingTenantId)?; - // - // let client_id = ClientId::new(client_id); - // let client_secret = ClientSecret::new(client_secret); - // - // let auth_url = AuthUrl::new(format!( - // "https://login.microsoftonline.com/{}/oauth2/v2.0/authorize", - // tenant_id.clone() - // )) - // .map_err(|e| ServerError::FailedToParseUrl(e))?; - // let token_url = TokenUrl::new(format!( - // "https://login.microsoftonline.com/{}/oauth2/v2.0/token", - // tenant_id.clone() - // )) - // .map_err(|e| ServerError::FailedToParseUrl(e))?; - // - // println!("Loading client"); - // let client = BasicClient::new( - // client_id.clone(), - // Some(client_secret.clone()), - // auth_url, - // Some(token_url), - // ) - // .set_redirect_uri( - // RedirectUrl::new(format!( - // "http://localhost:{}/auth/callback", - // port.to_string() - // )) - // .map_err(|e| ServerError::FailedToParseUrl(e))?, - // ); - // - // println!("Fetching JWKS"); - // let jwks = get_jwks() - // .await - // .map_err(|_| ServerError::FailedToFetchJWKS)?; - // - // let app_state = AppState { - // oauth_client: Arc::new(client), - // csrf_state: Arc::new(Mutex::new(HashMap::new())), - // pkce_verifier: Arc::new(Mutex::new(HashMap::new())), - // jwks: Arc::new(jwks), - // }; - // - // let token_middleware = TokenMiddleware::new(Arc::new(app_state.clone())); - // - // println!("Making app"); - // let app = Route::new() - // .at( - // "/", - // get(graphql_playground) - // .post(GraphQL::new(schema)) - // .with(token_middleware.clone()), - // ) - // .at("/health", get(health)) - // .at("/login", login.data(app_state.clone())) - // .at("/auth/callback", auth_callback.data(app_state.clone())) - // .at( - // "/verify", - // verify - // .data(app_state.clone()) - // .with(token_middleware.clone()), - // ) - // .at("/logout", logout.with(token_middleware.clone())) - // .with(CookieJarManager::new()) - // .with(Cors::new()); - // println!("App done"); - // Ok(app) - // } - /// Run the server on the default port until completion. pub async fn run(self) -> IoResult<()> { self.start().await?.wait().await @@ -357,13 +276,12 @@ impl RunningGraphServer { } } -async fn server_termination(mut internal_signal: Receiver<()>) { +async fn server_termination(mut internal_signal: Receiver<()>, tp: Option) { let ctrl_c = async { signal::ctrl_c() .await .expect("failed to install Ctrl+C handler"); }; - #[cfg(unix)] let terminate = async { signal::unix::signal(signal::unix::SignalKind::terminate()) @@ -371,21 +289,30 @@ async fn server_termination(mut internal_signal: Receiver<()>) { .recv() .await; }; - #[cfg(not(unix))] let terminate = std::future::pending::<()>(); let internal_terminate = async { internal_signal.recv().await; }; - tokio::select! { _ = ctrl_c => {}, _ = terminate => {}, _ = internal_terminate => {}, } - - opentelemetry::global::shutdown_tracer_provider(); + match tp { + None => {} + Some(p) => { + task::spawn_blocking(move || { + let res = p.shutdown(); + if let Err(e) = res { + debug!("Failed to shut down tracing provider: {:?}", e); + } + }) + .await + .unwrap(); + } + } } #[cfg(test)] @@ -394,16 +321,19 @@ mod server_tests { use crate::server::GraphServer; use chrono::prelude::*; + use raphtory_api::core::utils::logging::global_info_logger; use tokio::time::{sleep, Duration}; + use tracing::info; #[tokio::test] async fn test_server_start_stop() { + global_info_logger(); let tmp_dir = tempfile::tempdir().unwrap(); let server = GraphServer::new(tmp_dir.path().to_path_buf(), None, None).unwrap(); - println!("Calling start at time {}", Local::now()); + info!("Calling start at time {}", Local::now()); let handler = server.start_with_port(0); sleep(Duration::from_secs(1)).await; - println!("Calling stop at time {}", Local::now()); + info!("Calling stop at time {}", Local::now()); handler.await.unwrap().stop().await } } diff --git a/raphtory-graphql/src/server_config.rs b/raphtory-graphql/src/server_config.rs deleted file mode 100644 index 65d559ffdb..0000000000 --- a/raphtory-graphql/src/server_config.rs +++ /dev/null @@ -1,226 +0,0 @@ -use config::{Config, ConfigError, File}; -use serde::Deserialize; -use std::path::PathBuf; - -#[derive(Debug, Deserialize, PartialEq, Clone)] -pub struct LoggingConfig { - pub log_level: String, -} - -impl Default for LoggingConfig { - fn default() -> Self { - Self { - log_level: "INFO".to_string(), - } - } -} - -#[derive(Debug, Deserialize, PartialEq, Clone)] -pub struct CacheConfig { - pub capacity: u64, - pub tti_seconds: u64, -} - -impl Default for CacheConfig { - fn default() -> Self { - Self { - capacity: 30, - tti_seconds: 900, - } - } -} - -#[derive(Debug, Deserialize, PartialEq, Clone)] -pub struct AuthConfig { - pub client_id: Option, - pub client_secret: Option, - pub tenant_id: Option, -} - -impl Default for AuthConfig { - fn default() -> Self { - Self { - client_id: None, - client_secret: None, - tenant_id: None, - } - } -} - -#[derive(Debug, Deserialize, PartialEq, Clone)] -pub struct AppConfig { - pub logging: LoggingConfig, - pub cache: CacheConfig, - // pub auth: AuthConfig, -} - -impl Default for AppConfig { - fn default() -> Self { - Self { - logging: LoggingConfig::default(), - cache: CacheConfig::default(), - // auth: AuthConfig::default(), - } - } -} - -pub struct AppConfigBuilder { - logging: LoggingConfig, - cache: CacheConfig, - // auth: AuthConfig, -} - -impl AppConfigBuilder { - pub fn new() -> Self { - Self { - logging: LoggingConfig::default(), - cache: CacheConfig::default(), - // auth: AuthConfig::default(), - } - } - - pub fn from(config: AppConfig) -> Self { - Self { - logging: config.logging, - cache: config.cache, - // auth: config.auth, - } - } - - pub fn with_log_level(mut self, log_level: String) -> Self { - self.logging.log_level = log_level; - self - } - - pub fn with_cache_capacity(mut self, cache_capacity: u64) -> Self { - self.cache.capacity = cache_capacity; - self - } - - pub fn with_cache_tti_seconds(mut self, tti_seconds: u64) -> Self { - self.cache.tti_seconds = tti_seconds; - self - } - - // pub fn with_auth_client_id(mut self, client_id: String) -> Self { - // self.auth.client_id = Some(client_id); - // self - // } - // - // pub fn with_auth_client_secret(mut self, client_secret: String) -> Self { - // self.auth.client_secret = Some(client_secret); - // self - // } - // - // pub fn with_auth_tenant_id(mut self, tenant_id: String) -> Self { - // self.auth.tenant_id = Some(tenant_id); - // self - // } - - pub fn build(self) -> AppConfig { - AppConfig { - logging: self.logging, - cache: self.cache, - // auth: self.auth, - } - } -} - -// Order of precedence of config loading: config args >> config path >> config default -// Note: Since config args takes precedence over config path, ensure not to provide config args when starting server from a compile rust instance. -// This would cause configs from config paths to be ignored. The reason it has been implemented so is to avoid having to pass all the configs as -// args from the python instance i.e., being able to provide configs from config path as default configs and yet give precedence to config args. -pub fn load_config( - app_config: Option, - config_path: Option, -) -> Result { - let mut settings_config_builder = Config::builder(); - if let Some(config_path) = config_path { - settings_config_builder = settings_config_builder.add_source(File::from(config_path)); - } - let settings = settings_config_builder.build()?; - - let mut app_config_builder = if let Some(app_config) = app_config { - AppConfigBuilder::from(app_config) - } else { - AppConfigBuilder::new() - }; - - // Override with provided configs from config file if any - if let Some(log_level) = settings.get::("logging.log_level").ok() { - app_config_builder = app_config_builder.with_log_level(log_level); - } - if let Some(cache_capacity) = settings.get::("cache.capacity").ok() { - app_config_builder = app_config_builder.with_cache_capacity(cache_capacity); - } - if let Some(cache_tti_seconds) = settings.get::("cache.tti_seconds").ok() { - app_config_builder = app_config_builder.with_cache_tti_seconds(cache_tti_seconds); - } - // if let Some(client_id) = settings.get::("auth.client_id").ok() { - // app_config_builder = app_config_builder.with_auth_client_id(client_id); - // } - // if let Some(client_secret) = settings.get::("auth.client_secret").ok() { - // app_config_builder = app_config_builder.with_auth_client_secret(client_secret); - // } - // if let Some(tenant_id) = settings.get::("auth.tenant_id").ok() { - // app_config_builder = app_config_builder.with_auth_tenant_id(tenant_id); - // } - - Ok(app_config_builder.build()) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - #[test] - fn test_load_config_from_toml() { - let config_toml = r#" - [logging] - log_level = "DEBUG" - - [cache] - tti_seconds = 1000 - "#; - let config_path = PathBuf::from("test_config.toml"); - fs::write(&config_path, config_toml).unwrap(); - - let result = load_config(None, Some(config_path.clone())); - let expected_config = AppConfigBuilder::new() - .with_log_level("DEBUG".to_string()) - .with_cache_capacity(30) - .with_cache_tti_seconds(1000) - .build(); - - assert_eq!(result.unwrap(), expected_config); - - // Cleanup: delete the test TOML file - fs::remove_file(config_path).unwrap(); - } - - #[test] - fn test_load_config_with_custom_cache() { - let app_config = AppConfigBuilder::new() - .with_cache_capacity(50) - .with_cache_tti_seconds(1200) - .build(); - - let result = load_config(Some(app_config.clone()), None); - - assert_eq!(result.unwrap(), app_config); - } - - #[test] - fn test_load_config_with_custom_auth() { - let app_config = AppConfigBuilder::new() - // .with_auth_client_id("custom_client_id".to_string()) - // .with_auth_client_secret("custom_client_secret".to_string()) - // .with_auth_tenant_id("custom_tenant_id".to_string()) - .build(); - - let result = load_config(Some(app_config.clone()), None); - - assert_eq!(result.unwrap(), app_config); - } -} diff --git a/raphtory/Cargo.toml b/raphtory/Cargo.toml index a4eb00774d..a2c86acbe9 100644 --- a/raphtory/Cargo.toml +++ b/raphtory/Cargo.toml @@ -41,6 +41,7 @@ ouroboros = { workspace = true } either = { workspace = true } kdam = { workspace = true } bytemuck = { workspace = true } +tracing = {workspace = true} # io optional dependencies csv = { workspace = true, optional = true } @@ -75,7 +76,6 @@ pometry-storage = { workspace = true, optional = true } prost = { workspace = true, optional = true } prost-types = { workspace = true, optional = true } - [target.'cfg(target_os = "macos")'.dependencies] snmalloc-rs = { workspace = true } diff --git a/raphtory/src/algorithms/algorithm_result.rs b/raphtory/src/algorithms/algorithm_result.rs index 2083a6db16..02272a6cee 100644 --- a/raphtory/src/algorithms/algorithm_result.rs +++ b/raphtory/src/algorithms/algorithm_result.rs @@ -418,7 +418,8 @@ mod algorithm_result_test { prelude::{NO_PROPS, *}, }; use ordered_float::OrderedFloat; - use raphtory_api::core::entities::GID; + use raphtory_api::core::{entities::GID, utils::logging::global_info_logger}; + use tracing::info; fn create_algo_result_u64() -> AlgorithmResult { let g = create_graph(); @@ -671,10 +672,11 @@ mod algorithm_result_test { #[test] fn test_get_all() { + global_info_logger(); let algo_result = create_algo_result_u64(); let gotten_all = algo_result.get_all(); let names: Vec = gotten_all.keys().map(|vv| vv.name()).collect(); - println!("{:?}", names) + info!("{:?}", names) } #[test] diff --git a/raphtory/src/algorithms/community_detection/louvain.rs b/raphtory/src/algorithms/community_detection/louvain.rs index a2c7c39724..6b2a4cfed8 100644 --- a/raphtory/src/algorithms/community_detection/louvain.rs +++ b/raphtory/src/algorithms/community_detection/louvain.rs @@ -70,6 +70,8 @@ mod test { test_storage, }; use proptest::prelude::*; + use raphtory_api::core::utils::logging::global_info_logger; + use tracing::info; #[test] fn test_louvain() { @@ -136,7 +138,7 @@ mod test { use crate::io::csv_loader::CsvLoader; use serde::{Deserialize, Serialize}; use std::path::PathBuf; - + global_info_logger(); let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); d.push("resources/test"); let loader = CsvLoader::new(d.join("test.csv")).set_delimiter(","); @@ -156,7 +158,7 @@ mod test { test_storage!(&graph, |graph| { let result = louvain::(graph, 1.0, None, None); - println!("{result:?}") + info!("{result:?}") }); } } diff --git a/raphtory/src/algorithms/community_detection/modularity.rs b/raphtory/src/algorithms/community_detection/modularity.rs index 548f99ab5d..6c754a7be1 100644 --- a/raphtory/src/algorithms/community_detection/modularity.rs +++ b/raphtory/src/algorithms/community_detection/modularity.rs @@ -423,9 +423,12 @@ mod test { prelude::*, test_storage, }; + use raphtory_api::core::utils::logging::global_info_logger; + use tracing::info; #[test] fn test_delta() { + global_info_logger(); let graph = Graph::new(); graph.add_edge(0, 1, 2, NO_PROPS, None).unwrap(); graph.add_edge(0, 2, 1, NO_PROPS, None).unwrap(); @@ -441,7 +444,7 @@ mod test { let old_value = m.value(); assert_eq!(old_value, -0.5); let delta = m.move_delta(&VID(0), ComID(1)); - println!("delta: {delta}"); + info!("delta: {delta}"); m.move_node(&VID(0), ComID(1)); assert_eq!(m.value(), old_value + delta) }); @@ -449,6 +452,7 @@ mod test { #[test] fn test_aggregation() { + global_info_logger(); let graph = Graph::new(); graph.add_edge(0, 0, 1, NO_PROPS, None).unwrap(); graph.add_edge(0, 1, 0, NO_PROPS, None).unwrap(); @@ -463,7 +467,7 @@ mod test { let value_before = m.value(); let _ = m.aggregate(); let value_after = m.value(); - println!("before: {value_before}, after: {value_after}"); + info!("before: {value_before}, after: {value_after}"); assert_eq!(value_after, value_before); let delta = m.move_delta(&VID(0), ComID(1)); m.move_node(&VID(0), ComID(1)); diff --git a/raphtory/src/algorithms/components/lcc.rs b/raphtory/src/algorithms/components/lcc.rs index a6ab25501f..13398a693d 100644 --- a/raphtory/src/algorithms/components/lcc.rs +++ b/raphtory/src/algorithms/components/lcc.rs @@ -1,4 +1,5 @@ use raphtory_api::core::entities::GID; +use tracing::warn; use crate::{ algorithms::components::connected_components::weakly_connected_components, @@ -49,7 +50,7 @@ impl LargestConnectedComponent for Graph { } } if is_tie { - println!("Warning: The graph has two or more connected components that are both the largest. \ + warn!("Warning: The graph has two or more connected components that are both the largest. \ The returned component has been picked arbitrarily."); } return match connected_components_map.remove(&lcc_key) { diff --git a/raphtory/src/algorithms/dynamics/temporal/epidemics.rs b/raphtory/src/algorithms/dynamics/temporal/epidemics.rs index 83c20e90b1..1e46fac011 100644 --- a/raphtory/src/algorithms/dynamics/temporal/epidemics.rs +++ b/raphtory/src/algorithms/dynamics/temporal/epidemics.rs @@ -250,10 +250,12 @@ mod test { }; use rand::{rngs::SmallRng, Rng, SeedableRng}; use rand_distr::{Distribution, Exp}; + use raphtory_api::core::utils::logging::global_info_logger; use rayon::prelude::*; use stats::{mean, stddev}; #[cfg(feature = "storage")] use tempfile::TempDir; + use tracing::info; fn correct_res(x: f64) -> f64 { (1176. * x.powi(10) @@ -341,12 +343,13 @@ mod test { let mean = mean(actual.iter().copied()); let dev = stddev(actual.iter().copied()) / (num_tries as f64).sqrt(); let expected = correct_res(scaled_infection_rate); - println!("mean: {mean}, expected: {expected}, dev: {dev}, infection rate: {scaled_infection_rate}"); + info!("mean: {mean}, expected: {expected}, dev: {dev}, infection rate: {scaled_infection_rate}"); assert!((mean - expected).abs() < 2. * dev) } #[test] fn test_small_graph_medium() { + global_info_logger(); let event_rate = 0.00000001; let recovery_rate = 0.000000001; let p = 0.3; @@ -356,6 +359,7 @@ mod test { #[test] fn test_small_graph_high() { + global_info_logger(); let event_rate = 0.00000001; let recovery_rate = 0.000000001; let p = 0.7; @@ -365,6 +369,7 @@ mod test { #[test] fn test_small_graph_low() { + global_info_logger(); let event_rate = 0.00000001; let recovery_rate = 0.00000001; let p = 0.1; diff --git a/raphtory/src/algorithms/motifs/local_temporal_three_node_motifs.rs b/raphtory/src/algorithms/motifs/local_temporal_three_node_motifs.rs index 9acfc9745c..499ffc358d 100644 --- a/raphtory/src/algorithms/motifs/local_temporal_three_node_motifs.rs +++ b/raphtory/src/algorithms/motifs/local_temporal_three_node_motifs.rs @@ -19,6 +19,7 @@ use num_traits::Zero; use raphtory_api::core::entities::VID; use rustc_hash::FxHashSet; use std::{collections::HashMap, mem, ops::Add, slice::Iter}; +use tracing::debug; /////////////////////////////////////////////////////// // State objects for three node motifs @@ -330,9 +331,9 @@ where let delta_len = deltas.len(); let ctx: Context = g.into(); - println!("Running triangle step"); + debug!("Running triangle step"); let triadic_motifs = triangle_motifs(g, deltas.clone(), threads); - println!("Running rest of motifs"); + debug!("Running rest of motifs"); let star_motif_step = ATask::new(move |evv: &mut EvalNodeView| { let two_nodes = twonode_motif_count(evv, deltas.clone()); @@ -400,6 +401,8 @@ mod motifs_test { prelude::NO_PROPS, test_storage, }; + use raphtory_api::core::utils::logging::global_debug_logger; + use tracing::info; fn load_graph(edges: Vec<(i64, u64, u64)>) -> Graph { let graph = Graph::new(); @@ -445,6 +448,7 @@ mod motifs_test { #[ignore] #[test] fn test_triangle_motif() { + global_debug_logger(); let ij_kj_ik = vec![(1, 1, 2), (2, 3, 2), (3, 1, 3)]; let g = load_graph(ij_kj_ik); let mc = temporal_three_node_motif(&g, vec![3], None) @@ -452,7 +456,7 @@ mod motifs_test { .map(|(k, v)| (k.clone(), v[0].clone())) .into_iter() .collect::>>(); - println!("{:?}", mc.get("3").unwrap()); + info!("{:?}", mc.get("3").unwrap()); let ij_ki_jk = vec![(1, 1, 2), (2, 3, 1), (3, 2, 3)]; let g = load_graph(ij_ki_jk); @@ -461,7 +465,7 @@ mod motifs_test { .map(|(k, v)| (k.clone(), v[0].clone())) .into_iter() .collect::>>(); - println!("{:?}", mc.get("3").unwrap()); + info!("{:?}", mc.get("3").unwrap()); let ij_jk_ik = vec![(1, 1, 2), (2, 2, 3), (3, 1, 3)]; let g = load_graph(ij_jk_ik); @@ -470,7 +474,7 @@ mod motifs_test { .map(|(k, v)| (k.clone(), v[0].clone())) .into_iter() .collect::>>(); - println!("{:?}", mc.get("3").unwrap()); + info!("{:?}", mc.get("3").unwrap()); let ij_ik_jk = vec![(1, 1, 2), (2, 1, 3), (3, 2, 3)]; let g = load_graph(ij_ik_jk); @@ -479,7 +483,7 @@ mod motifs_test { .map(|(k, v)| (k.clone(), v[0].clone())) .into_iter() .collect::>>(); - println!("{:?}", mc.get("3").unwrap()); + info!("{:?}", mc.get("3").unwrap()); let ij_kj_ki = vec![(1, 1, 2), (2, 3, 2), (3, 3, 1)]; let g = load_graph(ij_kj_ki); @@ -488,7 +492,7 @@ mod motifs_test { .map(|(k, v)| (k.clone(), v[0].clone())) .into_iter() .collect::>>(); - println!("{:?}", mc.get("3").unwrap()); + info!("{:?}", mc.get("3").unwrap()); let ij_ki_kj = vec![(1, 1, 2), (2, 3, 1), (3, 3, 2)]; let g = load_graph(ij_ki_kj); @@ -497,7 +501,7 @@ mod motifs_test { .map(|(k, v)| (k.clone(), v[0].clone())) .into_iter() .collect::>>(); - println!("{:?}", mc.get("3").unwrap()); + info!("{:?}", mc.get("3").unwrap()); let ij_jk_ki = vec![(1, 1, 2), (2, 2, 3), (3, 3, 1)]; let g = load_graph(ij_jk_ki); @@ -506,7 +510,7 @@ mod motifs_test { .map(|(k, v)| (k.clone(), v[0].clone())) .into_iter() .collect::>>(); - println!("{:?}", mc.get("3").unwrap()); + info!("{:?}", mc.get("3").unwrap()); let ij_ik_kj = vec![(1, 1, 2), (2, 1, 3), (3, 3, 2)]; let g = load_graph(ij_ik_kj); @@ -515,7 +519,7 @@ mod motifs_test { .map(|(k, v)| (k.clone(), v[0].clone())) .into_iter() .collect::>>(); - println!("{:?}", mc.get("3").unwrap()); + info!("{:?}", mc.get("3").unwrap()); } #[test] @@ -621,9 +625,10 @@ mod motifs_test { #[test] fn test_windowed_graph() { + global_debug_logger(); let g = load_sample_graph(); let g_windowed = g.before(11).after(0); - println! {"windowed graph has {:?} vertices",g_windowed.count_nodes()} + info! {"windowed graph has {:?} vertices",g_windowed.count_nodes()} let binding = temporal_three_node_motif(&g_windowed, Vec::from([10]), None); let actual = binding diff --git a/raphtory/src/algorithms/motifs/three_node_motifs.rs b/raphtory/src/algorithms/motifs/three_node_motifs.rs index b942b57b99..ee707a55e6 100644 --- a/raphtory/src/algorithms/motifs/three_node_motifs.rs +++ b/raphtory/src/algorithms/motifs/three_node_motifs.rs @@ -349,6 +349,8 @@ mod three_node_motifs_test { use super::{ init_tri_count, map2d, TriangleEdge, TwoNodeCounter, TwoNodeEvent, INCOMING, OUTGOING, }; + use raphtory_api::core::utils::logging::global_info_logger; + use tracing::info; #[test] fn map_test() { @@ -357,6 +359,7 @@ mod three_node_motifs_test { #[test] fn two_node_test() { + global_info_logger(); let events = vec![ TwoNodeEvent { dir: OUTGOING, @@ -377,11 +380,12 @@ mod three_node_motifs_test { count3d: [0; 8], }; twonc.execute(&events, 5); - println!("motifs are {:?}", twonc.count3d); + info!("motifs are {:?}", twonc.count3d); } #[test] fn triad_test() { + global_info_logger(); let events = [(true, 0, 1, 1, 1), (false, 1, 0, 1, 2), (false, 0, 0, 0, 3)] .iter() .map(|x| TriangleEdge { @@ -394,6 +398,6 @@ mod three_node_motifs_test { .collect::>(); let mut triangle_count = init_tri_count(3); triangle_count.execute(&events, 5); - println!("triangle motifs are {:?}", triangle_count.final_counts); + info!("triangle motifs are {:?}", triangle_count.final_counts); } } diff --git a/raphtory/src/algorithms/pathing/single_source_shortest_path.rs b/raphtory/src/algorithms/pathing/single_source_shortest_path.rs index a9185a80a8..725309df13 100644 --- a/raphtory/src/algorithms/pathing/single_source_shortest_path.rs +++ b/raphtory/src/algorithms/pathing/single_source_shortest_path.rs @@ -83,6 +83,8 @@ mod sssp_tests { db::{api::mutation::AdditionOps, graph::graph::Graph}, test_storage, }; + use raphtory_api::core::utils::logging::global_info_logger; + use tracing::info; fn load_graph(edges: Vec<(i64, u64, u64)>) -> Graph { let graph = Graph::new(); @@ -94,6 +96,7 @@ mod sssp_tests { #[test] fn test_sssp_1() { + global_info_logger(); let graph = load_graph(vec![ (0, 1, 2), (1, 1, 3), @@ -130,7 +133,7 @@ mod sssp_tests { ]); assert_eq!(results, expected); let binding = single_source_shortest_path(graph, 5, Some(4)); - println!("{:?}", binding.get_all_with_names()); + info!("{:?}", binding.get_all_with_names()); }); } } diff --git a/raphtory/src/algorithms/pathing/temporal_reachability.rs b/raphtory/src/algorithms/pathing/temporal_reachability.rs index 62a61d7ffc..e10c90c698 100644 --- a/raphtory/src/algorithms/pathing/temporal_reachability.rs +++ b/raphtory/src/algorithms/pathing/temporal_reachability.rs @@ -136,8 +136,6 @@ pub fn temporally_reachable_nodes( let step2 = ATask::new(move |evv| { let msgs = evv.read(&recv_tainted_msgs); - // println!("v = {}, msgs = {:?}, taint_history = {:?}", evv.global_id(), msgs, evv.read(&taint_history)); - if !msgs.is_empty() { evv.global_update(&tainted_nodes, evv.node); @@ -148,8 +146,6 @@ pub fn temporally_reachable_nodes( evv.update(&taint_history, msg.clone()); }); - // println!("v = {}, taint_history = {:?}", evv.global_id(), evv.read(&taint_history)); - if stop_nodes.is_empty() || !stop_nodes.contains(&evv.node) { let earliest = evv.read(&earliest_taint_time); for eev in evv.window(earliest, i64::MAX).out_edges() { diff --git a/raphtory/src/core/entities/nodes/structure/adjset.rs b/raphtory/src/core/entities/nodes/structure/adjset.rs index 9d4dcae021..7a972cd894 100644 --- a/raphtory/src/core/entities/nodes/structure/adjset.rs +++ b/raphtory/src/core/entities/nodes/structure/adjset.rs @@ -236,9 +236,12 @@ impl + Copy + Send + Sync> Ad mod tadjset_tests { use super::*; use quickcheck_macros::quickcheck; + use raphtory_api::core::utils::logging::global_info_logger; + use tracing::info; #[quickcheck] fn insert_fuzz(input: Vec) -> bool { + global_info_logger(); let mut ts: AdjSet = AdjSet::default(); for (e, i) in input.iter().enumerate() { @@ -248,8 +251,8 @@ mod tadjset_tests { let res = input.iter().all(|i| ts.find(*i).is_some()); if !res { let ts_vec: Vec<(usize, usize)> = ts.iter().collect(); - println!("Input: {:?}", input); - println!("TAdjSet: {:?}", ts_vec); + info!("Input: {:?}", input); + info!("TAdjSet: {:?}", ts_vec); } res } diff --git a/raphtory/src/db/graph/graph.rs b/raphtory/src/db/graph/graph.rs index 8440465e9a..25d8213a89 100644 --- a/raphtory/src/db/graph/graph.rs +++ b/raphtory/src/db/graph/graph.rs @@ -367,10 +367,12 @@ mod db_tests { use raphtory_api::core::{ entities::GID, storage::arc_str::{ArcStr, OptionAsStr}, + utils::logging::global_info_logger, }; use serde_json::Value; use std::collections::{HashMap, HashSet}; use tempfile::TempDir; + use tracing::{error, info}; #[test] fn test_empty_graph() { @@ -451,7 +453,7 @@ mod db_tests { let expected_len = vs.iter().map(|(_, v)| v).sorted().dedup().count(); for (t, v) in vs { g.add_node(t, v, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); } @@ -460,12 +462,13 @@ mod db_tests { #[quickcheck] fn add_node_gets_names(vs: Vec) -> bool { + global_info_logger(); let g = Graph::new(); let expected_len = vs.iter().sorted().dedup().count(); for (t, name) in vs.iter().enumerate() { g.add_node(t as i64, name.clone(), NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| info!("{:?}", err)) .ok(); } @@ -644,16 +647,17 @@ mod db_tests { #[test] fn props_with_layers() { + global_info_logger(); let g = Graph::new(); g.add_edge(0, "A", "B", NO_PROPS, None).unwrap(); let ed = g.edge("A", "B").unwrap(); ed.add_constant_properties(vec![("CCC", Prop::str("RED"))], None) .unwrap(); - println!("{:?}", ed.properties().constant().as_map()); + info!("{:?}", ed.properties().constant().as_map()); g.add_edge(0, "A", "B", NO_PROPS, Some("LAYERONE")).unwrap(); ed.add_constant_properties(vec![("CCC", Prop::str("BLUE"))], Some("LAYERONE")) .unwrap(); - println!("{:?}", ed.properties().constant().as_map()); + info!("{:?}", ed.properties().constant().as_map()); } #[test] @@ -804,6 +808,7 @@ mod db_tests { #[test] fn test_explode_layers_time() { + global_info_logger(); let g = Graph::new(); g.add_edge( 1, @@ -812,7 +817,7 @@ mod db_tests { vec![("duration".to_string(), Prop::U32(5))], Some("a"), ) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_edge( 2, @@ -821,7 +826,7 @@ mod db_tests { vec![("duration".to_string(), Prop::U32(5))], Some("a"), ) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_edge( 3, @@ -830,7 +835,7 @@ mod db_tests { vec![("duration".to_string(), Prop::U32(5))], Some("a"), ) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_edge( 4, @@ -839,10 +844,10 @@ mod db_tests { vec![("duration".to_string(), Prop::U32(6))], Some("b"), ) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_edge(5, 1, 2, NO_PROPS, Some("c")) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); assert_eq!(g.latest_time(), Some(5)); @@ -870,13 +875,14 @@ mod db_tests { #[test] fn time_test() { + global_info_logger(); let g = Graph::new(); assert_eq!(g.latest_time(), None); assert_eq!(g.earliest_time(), None); g.add_node(5, 1, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); assert_eq!(g.latest_time(), Some(5)); @@ -889,7 +895,7 @@ mod db_tests { assert_eq!(g.earliest_time(), Some(10)); g.add_node(5, 1, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); assert_eq!(g.latest_time(), Some(10)); assert_eq!(g.earliest_time(), Some(5)); @@ -1706,6 +1712,7 @@ mod db_tests { #[quickcheck] fn test_graph_temporal_props(str_props: HashMap) -> bool { + global_info_logger(); let g = Graph::new(); let (t0, t1) = (1, 2); @@ -1739,7 +1746,7 @@ mod db_tests { g.properties().temporal().get(name).unwrap().at(t1) == Some(value.clone()) }); if !check { - println!("failed time-specific comparison for {:?}", str_props); + error!("failed time-specific comparison for {:?}", str_props); return false; } let check = check @@ -1751,7 +1758,7 @@ mod db_tests { .collect::>() == t0_props; if !check { - println!("failed latest value comparison for {:?} at t0", str_props); + error!("failed latest value comparison for {:?} at t0", str_props); return false; } let check = check @@ -1764,7 +1771,7 @@ mod db_tests { == Some(ve.clone()) }); if !check { - println!("failed latest value comparison for {:?} at t1", str_props); + error!("failed latest value comparison for {:?} at t1", str_props); return false; } check @@ -2211,9 +2218,10 @@ mod db_tests { #[test] fn large_id_is_consistent() { + global_info_logger(); let g = Graph::new(); g.add_node(0, 10000000000000000006, NO_PROPS, None).unwrap(); - println!("names: {:?}", g.nodes().name().collect_vec()); + info!("names: {:?}", g.nodes().name().collect_vec()); assert!(g .nodes() .name() @@ -2237,10 +2245,11 @@ mod db_tests { edges: Vec<(u64, u64, Vec)>, offset: i64, ) -> bool { + global_info_logger(); let mut correct = true; let mut check = |condition: bool, message: String| { if !condition { - println!("Failed: {}", message); + error!("Failed: {}", message); } correct = correct && condition; }; diff --git a/raphtory/src/db/graph/views/deletion_graph.rs b/raphtory/src/db/graph/views/deletion_graph.rs index 3516c17459..91190a0acb 100644 --- a/raphtory/src/db/graph/views/deletion_graph.rs +++ b/raphtory/src/db/graph/views/deletion_graph.rs @@ -674,7 +674,8 @@ mod test_deletions { prelude::*, }; use itertools::Itertools; - use raphtory_api::core::entities::GID; + use raphtory_api::core::{entities::GID, utils::logging::global_info_logger}; + use tracing::info; #[test] fn test_nodes() { @@ -1185,6 +1186,7 @@ mod test_deletions { #[test] fn test_jira() { + global_info_logger(); let g = PersistentGraph::new(); g.add_edge(0, 1, 2, [("added", Prop::I64(0))], Some("assigned")) @@ -1211,7 +1213,7 @@ mod test_deletions { .map(|vv| vv.id()) .collect_vec(); - println!("windowed edges = {:?}", nodes); + info!("windowed edges = {:?}", nodes); let nodes = g .window(0, 1701786285758) @@ -1222,7 +1224,7 @@ mod test_deletions { .map(|vv| vv.name()) .collect_vec(); - println!("windowed nodes = {:?}", nodes); + info!("windowed nodes = {:?}", nodes); let nodes = g .at(1701786285758) @@ -1232,7 +1234,7 @@ mod test_deletions { .into_iter() .map(|vv| vv.id()) .collect_vec(); - println!("at edges = {:?}", nodes); + info!("at edges = {:?}", nodes); let nodes = g .at(1701786285758) @@ -1243,8 +1245,7 @@ mod test_deletions { .map(|vv| vv.id()) .collect_vec(); - println!("at nodes = {:?}", nodes); - // assert_eq!(g.window(1, 2).node(0).unwrap().out_degree(), 1) + info!("at nodes = {:?}", nodes); } #[test] diff --git a/raphtory/src/db/graph/views/window_graph.rs b/raphtory/src/db/graph/views/window_graph.rs index 4e49f6460f..0aeb0fce3b 100644 --- a/raphtory/src/db/graph/views/window_graph.rs +++ b/raphtory/src/db/graph/views/window_graph.rs @@ -556,10 +556,11 @@ mod views_test { use quickcheck::TestResult; use quickcheck_macros::quickcheck; use rand::prelude::*; - use raphtory_api::core::entities::GID; + use raphtory_api::core::{entities::GID, utils::logging::global_info_logger}; use rayon::prelude::*; #[cfg(feature = "storage")] use tempfile::TempDir; + use tracing::{error, info}; #[test] fn windowed_graph_nodes_degree() { @@ -640,6 +641,7 @@ mod views_test { #[test] fn graph_has_node_check_fail() { + global_info_logger(); let vs: Vec<(i64, u64)> = vec![ (1, 0), (-100, 262), @@ -653,7 +655,7 @@ mod views_test { for (t, v) in &vs { graph .add_node(*t, *v, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); } @@ -666,6 +668,7 @@ mod views_test { #[quickcheck] fn windowed_graph_has_node(mut vs: Vec<(i64, u64)>) -> TestResult { + global_info_logger(); if vs.is_empty() { return TestResult::discard(); } @@ -681,7 +684,7 @@ mod views_test { for (t, v) in &vs { g.add_node(*t, *v, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); } @@ -713,7 +716,8 @@ mod views_test { // FIXME: Issue #46 // #[quickcheck] // fn windowed_disk_graph_has_node(mut vs: Vec<(i64, u64)>) -> TestResult { - // if vs.is_empty() { + // global_info_logger(); + // if vs.is_empty() { // return TestResult::discard(); // } // @@ -727,7 +731,7 @@ mod views_test { // let g = Graph::new(); // for (t, v) in &vs { // g.add_node(*t, *v, NO_PROPS, None) - // .map_err(|err| println!("{:?}", err)) + // .map_err(|err| error!("{:?}", err)) // .ok(); // } // let test_dir = TempDir::new().unwrap(); @@ -857,6 +861,7 @@ mod views_test { mut edges: Vec<(i64, (u64, u64))>, window: Range, ) -> TestResult { + global_info_logger(); if window.end < window.start { return TestResult::discard(); } @@ -874,12 +879,12 @@ mod views_test { let wg = g.window(window.start, window.end); if wg.count_edges() != true_edge_count { - println!( + info!( "failed, g.num_edges() = {}, true count = {}", wg.count_edges(), true_edge_count ); - println!("g.edges() = {:?}", wg.edges().iter().collect_vec()); + info!("g.edges() = {:?}", wg.edges().iter().collect_vec()); } TestResult::from_bool(wg.count_edges() == true_edge_count) } @@ -1049,13 +1054,14 @@ mod views_test { #[test] fn test_algorithm_on_windowed_graph() { + global_info_logger(); let graph = Graph::new(); graph.add_edge(0, 1, 2, NO_PROPS, None).unwrap(); test_storage!(&graph, |graph| { let w = graph.window(0, 1); let res = degree_centrality(&w, None); - println!("{:?}", res) + info!("{:?}", res) }); } diff --git a/raphtory/src/disk_graph/mod.rs b/raphtory/src/disk_graph/mod.rs index 3126f83e36..00a9f11503 100644 --- a/raphtory/src/disk_graph/mod.rs +++ b/raphtory/src/disk_graph/mod.rs @@ -30,6 +30,7 @@ use pometry_storage::{ use raphtory_api::core::entities::edges::edge_ref::EdgeRef; use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use tracing::warn; pub mod graph_impl; pub mod storage_interface; @@ -254,7 +255,7 @@ impl DiskGraphStorage { .resolve_prop_id(prop_name, data_type.into(), false) .expect("Arrow data types should without failing"); if resolved_id != id { - println!("Warning: Layers with different edge properties are not supported by the high-level apis on top of the disk_graph graph yet, edge properties will not be available to high-level apis"); + warn!("Warning: Layers with different edge properties are not supported by the high-level apis on top of the disk_graph graph yet, edge properties will not be available to high-level apis"); edge_meta = Meta::new(); break; } diff --git a/raphtory/src/graph_loader/company_house.rs b/raphtory/src/graph_loader/company_house.rs index 9231fbae59..dda83c0146 100644 --- a/raphtory/src/graph_loader/company_house.rs +++ b/raphtory/src/graph_loader/company_house.rs @@ -2,6 +2,7 @@ use crate::{io::csv_loader::CsvLoader, prelude::*}; use chrono::DateTime; use serde::Deserialize; use std::{fs, path::PathBuf, time::Instant}; +use tracing::{error, info}; #[derive(Deserialize, std::fmt::Debug)] pub struct CompanyHouse { @@ -30,14 +31,14 @@ pub fn company_house_graph(path: Option) -> Graph { let now = Instant::now(); let g = Graph::decode(encoded_data_dir.as_path()) .map_err(|err| { - println!( + error!( "Restoring from bincode failed with error: {}! Reloading file!", err ) }) .ok()?; - println!( + info!( "Loaded graph from encoded data files {} with {} nodes, {} edges which took {} seconds", encoded_data_dir.to_str().unwrap(), g.count_nodes(), @@ -166,18 +167,21 @@ pub fn company_house_graph(path: Option) -> Graph { #[cfg(test)] mod company_house_graph_test { use crate::db::api::view::{NodeViewOps, TimeOps}; + use raphtory_api::core::utils::logging::global_info_logger; + use tracing::info; use super::*; #[test] #[ignore] fn test_ch_load() { + global_info_logger(); let g = company_house_graph(None); assert_eq!(g.start().unwrap(), 1000); assert_eq!(g.end().unwrap(), 1001); g.window(1000, 1001) .nodes() .into_iter() - .for_each(|v| println!("nodeid = {}", v.id())); + .for_each(|v| info!("nodeid = {}", v.id())); } } diff --git a/raphtory/src/graph_loader/karate_club.rs b/raphtory/src/graph_loader/karate_club.rs index 0a037d40a9..5f365f8570 100644 --- a/raphtory/src/graph_loader/karate_club.rs +++ b/raphtory/src/graph_loader/karate_club.rs @@ -1,4 +1,5 @@ use crate::{db::api::mutation::AdditionOps, prelude::*}; +use tracing::error; /// `karate_club_graph` constructs a karate club graph. /// @@ -68,7 +69,7 @@ pub fn karate_club_graph() -> Graph { for i in 0..34 { graph .add_node(0, i, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); } @@ -83,7 +84,7 @@ pub fn karate_club_graph() -> Graph { let actual_row = row - 1; graph .add_edge(0, actual_row as u64, col as u64, [("weight", entry)], None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); } } @@ -97,7 +98,7 @@ pub fn karate_club_graph() -> Graph { const_prop = "Mr. Hi"; } node.add_constant_properties([("club", const_prop)]) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); } diff --git a/raphtory/src/graph_loader/lotr_graph.rs b/raphtory/src/graph_loader/lotr_graph.rs index 77ddf88552..5dcc3ee672 100644 --- a/raphtory/src/graph_loader/lotr_graph.rs +++ b/raphtory/src/graph_loader/lotr_graph.rs @@ -24,6 +24,7 @@ use crate::{graph_loader::fetch_file, io::csv_loader::CsvLoader, prelude::*}; use serde::Deserialize; use std::path::PathBuf; +use tracing::error; #[derive(Deserialize, std::fmt::Debug)] pub struct Lotr { @@ -66,10 +67,10 @@ pub fn lotr_graph() -> Graph { let time = lotr.time; g.add_node(time, src_id.clone(), NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_node(time, dst_id.clone(), NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_edge(time, src_id.clone(), dst_id.clone(), NO_PROPS, None) .expect("Error: Unable to add edge"); diff --git a/raphtory/src/graph_loader/mod.rs b/raphtory/src/graph_loader/mod.rs index 018cea4842..9d9ff124e7 100644 --- a/raphtory/src/graph_loader/mod.rs +++ b/raphtory/src/graph_loader/mod.rs @@ -168,6 +168,8 @@ fn unzip_file(zip_file_path: &str, destination_path: &str) -> std::io::Result<() mod graph_loader_test { use crate::{graph_loader::fetch_file, prelude::*}; use csv::StringRecord; + use raphtory_api::core::utils::logging::global_info_logger; + use tracing::info; #[test] fn test_fetch_file() { @@ -268,6 +270,7 @@ mod graph_loader_test { #[test] fn test_all_neighbours_window() { + global_info_logger(); let g = crate::graph_loader::lotr_graph::lotr_graph(); assert_eq!(g.count_edges(), 701); @@ -280,7 +283,7 @@ mod graph_loader_test { .neighbours() .iter() { - println!("{:?}", v.id()) + info!("{:?}", v.id()) } assert_eq!( g.node("Gandalf") diff --git a/raphtory/src/graph_loader/reddit_hyperlinks.rs b/raphtory/src/graph_loader/reddit_hyperlinks.rs index 350f3600d0..73e90be827 100644 --- a/raphtory/src/graph_loader/reddit_hyperlinks.rs +++ b/raphtory/src/graph_loader/reddit_hyperlinks.rs @@ -46,6 +46,7 @@ use std::{ io::{self, BufRead}, path::{Path, PathBuf}, }; +use tracing::error; /// Download the dataset and return the path to the file /// # Arguments @@ -137,16 +138,16 @@ pub fn generate_reddit_graph(path: PathBuf) -> Graph { ), ]; g.add_node(time, *src_id, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_node(time, *dst_id, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_edge(time, *src_id, *dst_id, edge_properties, None) .expect("Error: Unable to add edge"); } Err(e) => { - println!("{}", e) + error!("{}", e) } } } diff --git a/raphtory/src/graph_loader/stable_coins.rs b/raphtory/src/graph_loader/stable_coins.rs index 1f31ff82c2..bd7a0b5ac6 100644 --- a/raphtory/src/graph_loader/stable_coins.rs +++ b/raphtory/src/graph_loader/stable_coins.rs @@ -7,6 +7,7 @@ use chrono::DateTime; use regex::Regex; use serde::Deserialize; use std::{collections::HashMap, fs, path::PathBuf, time::Instant}; +use tracing::{error, info}; #[allow(dead_code)] #[derive(Deserialize, std::fmt::Debug)] @@ -40,14 +41,14 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { let now = Instant::now(); let g = Graph::decode(encoded_data_file.as_path()) .map_err(|err| { - println!( + error!( "Restoring from bincode failed with error: {}! Reloading file!", err ) }) .ok()?; - println!( + info!( "Loaded graph from encoded data files {} with {} nodes, {} edges which took {} seconds", encoded_data_file.to_str().unwrap(), g.count_nodes(), @@ -98,7 +99,7 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { }) .expect("Failed to load graph from CSV data files"); - println!( + info!( "Loaded graph from CSV data files {} with {} nodes, {} edges which took {} seconds", encoded_data_dir.to_str().unwrap(), g.count_nodes(), diff --git a/raphtory/src/graphgen/preferential_attachment.rs b/raphtory/src/graphgen/preferential_attachment.rs index 72a992d095..e93b4122f0 100644 --- a/raphtory/src/graphgen/preferential_attachment.rs +++ b/raphtory/src/graphgen/preferential_attachment.rs @@ -12,6 +12,7 @@ //! ba_preferential_attachment(&graph, 1000, 10, None); //! ``` +use super::next_id; use crate::{ db::{ api::{mutation::AdditionOps, view::*}, @@ -21,8 +22,7 @@ use crate::{ }; use rand::{rngs::StdRng, Rng, SeedableRng}; use std::collections::HashSet; - -use super::next_id; +use tracing::error; /// Generates a graph using the preferential attachment model. /// @@ -75,7 +75,7 @@ pub fn ba_preferential_attachment( max_id = next_id(view, Some(max_id)); graph .add_node(latest_time, &max_id, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); degrees.push(0); ids.push(max_id.clone()); @@ -130,6 +130,7 @@ pub fn ba_preferential_attachment( mod preferential_attachment_tests { use super::*; use crate::graphgen::random_attachment::random_attachment; + use raphtory_api::core::utils::logging::global_info_logger; #[test] fn blank_graph() { let graph = Graph::new(); @@ -140,11 +141,12 @@ mod preferential_attachment_tests { #[test] fn only_nodes() { + global_info_logger(); let graph = Graph::new(); for i in 0..10 { graph .add_node(i, i as u64, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); } diff --git a/raphtory/src/graphgen/random_attachment.rs b/raphtory/src/graphgen/random_attachment.rs index 00df8f2dd2..9822d9f634 100644 --- a/raphtory/src/graphgen/random_attachment.rs +++ b/raphtory/src/graphgen/random_attachment.rs @@ -21,6 +21,7 @@ use crate::{ prelude::{NodeStateOps, NO_PROPS}, }; use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng}; +use tracing::error; use super::next_id; @@ -68,7 +69,7 @@ pub fn random_attachment( latest_time += 1; graph .add_node(latest_time, &max_id, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); ids.push(max_id.clone()); } @@ -90,6 +91,7 @@ pub fn random_attachment( mod random_graph_test { use super::*; use crate::graphgen::preferential_attachment::ba_preferential_attachment; + use raphtory_api::core::utils::logging::global_info_logger; #[test] fn blank_graph() { let graph = Graph::new(); @@ -100,11 +102,12 @@ mod random_graph_test { #[test] fn only_nodes() { + global_info_logger(); let graph = Graph::new(); for i in 0..10 { graph .add_node(i, i as u64, NO_PROPS, None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); } diff --git a/raphtory/src/io/csv_loader.rs b/raphtory/src/io/csv_loader.rs index 1baa0fdd1a..26d8382b3f 100644 --- a/raphtory/src/io/csv_loader.rs +++ b/raphtory/src/io/csv_loader.rs @@ -3,17 +3,19 @@ //! # Example //! ```no_run //! use std::path::{Path, PathBuf}; +//! use tracing::{error, info}; //! use regex::Regex; //! use raphtory::io::csv_loader::CsvLoader; //! use raphtory::graph_loader::lotr_graph::Lotr; //! use raphtory::prelude::*; -//! +//! use raphtory::logging::global_info_logger; +//! global_info_logger(); //! let g = Graph::new(); //! let csv_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "../../resource/"] //! .iter() //! .collect(); //! -//! println!("path = {}", csv_path.as_path().to_str().unwrap()); +//! info!("path = {}", csv_path.as_path().to_str().unwrap()); //! let csv_loader = CsvLoader::new(Path::new(&csv_path)); //! let has_header = true; //! let r = Regex::new(r".+(lotr.csv)").unwrap(); @@ -34,7 +36,7 @@ //! [("name", Prop::str("Character"))], //! None, //! ) -//! .map_err(|err| println!("{:?}", err)) +//! .map_err(|err| error!("{:?}", err)) //! .ok(); //! g.add_node( //! time, @@ -42,7 +44,7 @@ //! [("name", Prop::str("Character"))], //! None, //! ) -//! .map_err(|err| println!("{:?}", err)) +//! .map_err(|err| error!("{:?}", err)) //! .ok(); //! g.add_edge( //! time, @@ -77,6 +79,7 @@ use std::{ io::BufReader, path::{Path, PathBuf}, }; +use tracing::info; #[derive(Debug)] pub enum CsvErr { @@ -394,7 +397,7 @@ impl CsvLoader { { let file_path: PathBuf = path.into(); if self.print_file_name { - println!("Loading file: {:?}", file_path); + info!("Loading file: {:?}", file_path); } let mut csv_reader = self.csv_reader(file_path)?; let records_iter = csv_reader.deserialize::(); @@ -475,9 +478,11 @@ impl CsvLoader { mod csv_loader_test { use crate::{io::csv_loader::CsvLoader, prelude::*}; use csv::StringRecord; + use raphtory_api::core::utils::logging::global_info_logger; use regex::Regex; use serde::Deserialize; use std::path::{Path, PathBuf}; + use tracing::{error, info}; #[test] fn regex_match() { @@ -519,10 +524,10 @@ mod csv_loader_test { let time = lotr.time; g.add_node(time, src_id, [("name", Prop::str("Character"))], None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_node(time, dst_id, [("name", Prop::str("Character"))], None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_edge( time, @@ -547,10 +552,10 @@ mod csv_loader_test { let time = lotr.get(2).map(|s| s.parse::().unwrap()).unwrap(); g.add_node(time, src_id, [("name", Prop::str("Character"))], None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_node(time, dst_id, [("name", Prop::str("Character"))], None) - .map_err(|err| println!("{:?}", err)) + .map_err(|err| error!("{:?}", err)) .ok(); g.add_edge( time, @@ -566,13 +571,14 @@ mod csv_loader_test { #[test] fn test_headers_flag_and_delimiter() { + global_info_logger(); let g = Graph::new(); // todo: move file path to data module let csv_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "../resource/"] .iter() .collect(); - println!("path = {}", csv_path.as_path().to_str().unwrap()); + info!("path = {}", csv_path.as_path().to_str().unwrap()); let csv_loader = CsvLoader::new(Path::new(&csv_path)); let has_header = true; let r = Regex::new(r".+(lotr.csv)").unwrap(); @@ -587,6 +593,7 @@ mod csv_loader_test { #[test] #[should_panic] fn test_wrong_header_flag_file_with_header() { + global_info_logger(); let g = Graph::new(); // todo: move file path to data module let csv_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "../../resource/"] @@ -602,6 +609,7 @@ mod csv_loader_test { #[test] #[should_panic] fn test_flag_has_header_but_file_has_no_header() { + global_info_logger(); let g = Graph::new(); // todo: move file path to data module let csv_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "../../resource/"] @@ -617,6 +625,7 @@ mod csv_loader_test { #[test] #[should_panic] fn test_wrong_header_names() { + global_info_logger(); let g = Graph::new(); // todo: move file path to data module let csv_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "../../resource/"] @@ -632,6 +641,7 @@ mod csv_loader_test { #[test] #[should_panic] fn test_wrong_delimiter() { + global_info_logger(); let g = Graph::new(); // todo: move file path to data module let csv_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "../../resource/"] diff --git a/raphtory/src/io/json_loader.rs b/raphtory/src/io/json_loader.rs index 2984b87759..9c472f418e 100644 --- a/raphtory/src/io/json_loader.rs +++ b/raphtory/src/io/json_loader.rs @@ -16,6 +16,7 @@ use std::{ io::BufReader, path::{Path, PathBuf}, }; +use tracing::{error, info}; #[derive(Debug)] pub enum JsonErr { @@ -269,7 +270,7 @@ impl JsonLinesLoader { { let file_path: PathBuf = path.into(); if self.print_file_name { - println!("Loading file: {:?}", file_path); + info!("Loading file: {:?}", file_path); } let json_reader = self.json_reader(file_path)?; @@ -281,7 +282,7 @@ impl JsonLinesLoader { if let Ok(record) = rec { loader(record, g)? } else { - println!("Error parsing record: {:?}", rec); + error!("Error parsing record: {:?}", rec); } } diff --git a/raphtory/src/lib.rs b/raphtory/src/lib.rs index c536653ce0..3676f38d60 100644 --- a/raphtory/src/lib.rs +++ b/raphtory/src/lib.rs @@ -141,6 +141,8 @@ pub mod prelude { #[cfg(feature = "storage")] pub use polars_arrow as arrow2; +pub use raphtory_api::core::utils::logging; + #[cfg(test)] mod test_utils { use crate::prelude::*; diff --git a/raphtory/src/python/graph/io/pandas_loaders.rs b/raphtory/src/python/graph/io/pandas_loaders.rs index c8a7114b77..55327a4250 100644 --- a/raphtory/src/python/graph/io/pandas_loaders.rs +++ b/raphtory/src/python/graph/io/pandas_loaders.rs @@ -15,6 +15,7 @@ use pyo3::{ types::{IntoPyDict, PyDict}, }; use std::collections::HashMap; +use tracing::error; pub fn load_nodes_from_pandas< G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps, @@ -299,7 +300,7 @@ except NameError: "#; if let Err(e) = py.run(code, None, None) { - println!("Error checking if running in a jupyter notebook: {}", e); + error!("Error checking if running in a jupyter notebook: {}", e); return; } @@ -310,7 +311,7 @@ except NameError: } } Err(e) => { - println!("Error checking if running in a jupyter notebook: {}", e); + error!("Error checking if running in a jupyter notebook: {}", e); } }; } diff --git a/raphtory/src/python/graph/properties/props.rs b/raphtory/src/python/graph/properties/props.rs index 43fb42f97b..2057820cc3 100644 --- a/raphtory/src/python/graph/properties/props.rs +++ b/raphtory/src/python/graph/properties/props.rs @@ -46,7 +46,6 @@ impl<'source> FromPyObject<'source> for PyPropsComp { } else if let Ok(m) = ob.extract::>() { Ok(PyPropsComp(m)) } else { - println!("sp4"); Err(PyTypeError::new_err("not comparable with properties")) } } diff --git a/raphtory/src/search/mod.rs b/raphtory/src/search/mod.rs index 66427b98c6..73d7b674e4 100644 --- a/raphtory/src/search/mod.rs +++ b/raphtory/src/search/mod.rs @@ -944,10 +944,11 @@ impl DeletionOps for IndexedGraph {} #[cfg(test)] mod test { + use super::*; + use raphtory_api::core::utils::logging::global_info_logger; use std::time::SystemTime; use tantivy::{doc, DocAddress, Order}; - - use super::*; + use tracing::info; #[test] fn index_numeric_props() { @@ -981,6 +982,7 @@ mod test { #[cfg(feature = "proto")] #[ignore = "this test is for experiments with the jira graph"] fn load_jira_graph() -> Result<(), GraphError> { + global_info_logger(); let graph = Graph::decode("/tmp/graphs/jira").expect("failed to load graph"); assert!(graph.count_nodes() > 0); @@ -988,14 +990,14 @@ mod test { let index_graph: IndexedGraph = graph.into(); let elapsed = now.elapsed().unwrap().as_secs(); - println!("indexing took: {:?}", elapsed); + info!("indexing took: {:?}", elapsed); let issues = index_graph.search_nodes("name:'DEV-1690'", 5, 0)?; assert!(!issues.is_empty()); let names = issues.into_iter().map(|v| v.name()).collect::>(); - println!("names: {:?}", names); + info!("names: {:?}", names); Ok(()) } diff --git a/raphtory/src/serialise/incremental.rs b/raphtory/src/serialise/incremental.rs index bb1f9192ec..0937b763dd 100644 --- a/raphtory/src/serialise/incremental.rs +++ b/raphtory/src/serialise/incremental.rs @@ -28,6 +28,7 @@ use std::{ path::Path, sync::Arc, }; +use tracing::instrument; #[derive(Debug)] pub struct GraphWriter { @@ -294,7 +295,7 @@ impl CacheOps for G { self.encode(path.as_ref())?; self.init_cache(path) } - + #[instrument(level = "debug", skip(self))] fn write_updates(&self) -> Result<(), GraphError> { let cache = self.get_cache().ok_or(GraphError::CacheNotInnitialised)?; cache.write() @@ -313,19 +314,20 @@ mod test { use raphtory_api::core::{ entities::{GidRef, VID}, storage::dict_mapper::MaybeNew, + utils::logging::global_info_logger, }; use std::fs::File; use tempfile::NamedTempFile; #[test] fn test_write_failure() { + global_info_logger(); let tmp_file = NamedTempFile::new().unwrap(); let read_only = File::open(tmp_file.path()).unwrap(); let cache = GraphWriter::new(read_only); cache.resolve_node(MaybeNew::New(VID(0)), GidRef::Str("0")); let res = cache.write(); - println!("{res:?}"); assert!(res.is_err()); assert_eq!(cache.proto_delta.lock().nodes.len(), 1); } diff --git a/raphtory/src/serialise/serialise.rs b/raphtory/src/serialise/serialise.rs index 6822abb13a..bc188a4e4f 100644 --- a/raphtory/src/serialise/serialise.rs +++ b/raphtory/src/serialise/serialise.rs @@ -541,7 +541,8 @@ mod proto_test { }; use chrono::{DateTime, NaiveDateTime}; use proptest::proptest; - use raphtory_api::core::storage::arc_str::ArcStr; + use raphtory_api::core::{storage::arc_str::ArcStr, utils::logging::global_info_logger}; + use tracing::info; #[test] fn node_no_props() { @@ -912,6 +913,7 @@ mod proto_test { #[test] fn test_incremental_writing_on_graph() { + global_info_logger(); let g = Graph::new(); let mut props = vec![]; write_props_to_vec(&mut props); @@ -948,16 +950,17 @@ mod proto_test { g.add_edge(7, "Bob", "Charlie", [("friends", false)], Some("two")) .unwrap(); g.write_updates().unwrap(); - println!("{g:?}"); + info!("{g:?}"); let g2 = Graph::decode(temp_cache_file.path()).unwrap(); - println!("{g2:?}"); + info!("{g2:?}"); assert_graph_equal(&g, &g2); } #[test] fn test_incremental_writing_on_persistent_graph() { + global_info_logger(); let g = PersistentGraph::new(); let mut props = vec![]; write_props_to_vec(&mut props); @@ -994,10 +997,10 @@ mod proto_test { g.add_edge(7, "Bob", "Charlie", [("friends", false)], Some("two")) .unwrap(); g.write_updates().unwrap(); - println!("{g:?}"); + info!("{g:?}"); let g2 = PersistentGraph::decode(temp_cache_file.path()).unwrap(); - println!("{g2:?}"); + info!("{g2:?}"); assert_graph_equal(&g, &g2); } diff --git a/raphtory/src/vectors/embeddings.rs b/raphtory/src/vectors/embeddings.rs index fcfef02ead..3ae308c5b5 100644 --- a/raphtory/src/vectors/embeddings.rs +++ b/raphtory/src/vectors/embeddings.rs @@ -4,9 +4,10 @@ use async_openai::{ Client, }; use itertools::Itertools; +use tracing::info; pub async fn openai_embedding(texts: Vec) -> Vec { - println!("computing embeddings for {} texts", texts.len()); + info!("computing embeddings for {} texts", texts.len()); let client = Client::new(); let request = CreateEmbeddingRequest { model: "text-embedding-ada-002".to_owned(), @@ -16,7 +17,7 @@ pub async fn openai_embedding(texts: Vec) -> Vec { dimensions: None, }; let response = client.embeddings().create(request).await.unwrap(); - println!("Generated embeddings successfully"); + info!("Generated embeddings successfully"); response .data .into_iter() @@ -27,7 +28,7 @@ pub async fn openai_embedding(texts: Vec) -> Vec { // this is currently commented out so we don't need to add any new dependencies // but might be potentially useful in the future // async fn sentence_transformers_embeddings(texts: Vec) -> Vec { -// println!("computing embeddings for {} texts", texts.len()); +// info!("computing embeddings for {} texts", texts.len()); // Python::with_gil(|py| { // let sentence_transformers = py.import("sentence_transformers")?; // let locals = [("sentence_transformers", sentence_transformers)].into_py_dict(py); diff --git a/raphtory/src/vectors/similarity_search_utils.rs b/raphtory/src/vectors/similarity_search_utils.rs index ee80067831..03bc058ea3 100644 --- a/raphtory/src/vectors/similarity_search_utils.rs +++ b/raphtory/src/vectors/similarity_search_utils.rs @@ -55,7 +55,6 @@ fn cosine(vector1: &Embedding, vector2: &Embedding) -> f32 { // see: https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use let normalized = dot_product / (x_length.sqrt() * y_length.sqrt()); - // println!("cosine for {vector1:?} and {vector2:?} is {normalized}"); assert!(normalized <= 1.001); assert!(normalized >= -1.001); normalized diff --git a/raphtory/src/vectors/template.rs b/raphtory/src/vectors/template.rs index 7a03798da7..5ea23fd6ab 100644 --- a/raphtory/src/vectors/template.rs +++ b/raphtory/src/vectors/template.rs @@ -14,6 +14,7 @@ use minijinja::{ use raphtory_api::core::storage::arc_str::ArcStr; use serde::Serialize; use std::sync::Arc; +use tracing::error; #[derive(Debug)] struct PropUpdate { @@ -181,7 +182,7 @@ impl DocumentTemplate { match template.render(GraphTemplateContext::from(graph)) { Ok(document) => Box::new(std::iter::once(document.into())), Err(error) => { - eprintln!("Template render failed for a node, skipping: {error}"); + error!("Template render failed for a node, skipping: {error}"); empty_iter() } } @@ -202,7 +203,7 @@ impl DocumentTemplate { match template.render(NodeTemplateContext::from(node)) { Ok(document) => Box::new(std::iter::once(document.into())), Err(error) => { - eprintln!("Template render failed for a node, skipping: {error}"); + error!("Template render failed for a node, skipping: {error}"); empty_iter() } } @@ -223,7 +224,7 @@ impl DocumentTemplate { match template.render(EdgeTemplateContext::from(edge)) { Ok(document) => Box::new(std::iter::once(document.into())), Err(error) => { - eprintln!("Template render failed for an edge, skipping: {error}"); + error!("Template render failed for an edge, skipping: {error}"); empty_iter() } } diff --git a/raphtory/src/vectors/vectorisable.rs b/raphtory/src/vectors/vectorisable.rs index 1d266b2d12..cb0c5e4c47 100644 --- a/raphtory/src/vectors/vectorisable.rs +++ b/raphtory/src/vectors/vectorisable.rs @@ -8,6 +8,7 @@ use crate::{ use async_trait::async_trait; use itertools::Itertools; use std::{collections::HashMap, path::PathBuf}; +use tracing::info; const CHUNK_SIZE: usize = 1000; @@ -90,7 +91,7 @@ impl Vectorisable for G { let cache_storage = cache.map(EmbeddingCache::from_path); if verbose { - println!("computing embeddings for graph"); + info!("computing embeddings for graph"); } let graph_ref_map = compute_embedding_groups(graph_docs, embedding.as_ref(), &cache_storage).await; @@ -101,13 +102,13 @@ impl Vectorisable for G { .unwrap_or_else(|| vec![]); // there should be only one value here, TODO: check that's true if verbose { - println!("computing embeddings for nodes"); + info!("computing embeddings for nodes"); } let node_refs = compute_embedding_groups(nodes_iter, embedding.as_ref(), &cache_storage).await; if verbose { - println!("computing embeddings for edges"); + info!("computing embeddings for edges"); } let edge_refs = compute_embedding_groups(edges_iter, embedding.as_ref(), &cache_storage).await; // FIXME: re-enable