Merge master into disk_merge

.github/workflows/_release_python.yml

@@ -24,6 +24,10 @@ jobs:
           permission: "write"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - name: Set up cargo cache
         uses: actions/cache@v3
         continue-on-error: false
@@ -121,6 +125,10 @@ jobs:
           permission: "write"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: actions/checkout@v3
         with:
           ref: ${{ inputs.base }}
@@ -157,6 +165,10 @@ jobs:
           permission: "write"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - name: Set up cargo cache
         uses: actions/cache@v3
         continue-on-error: false

.github/workflows/_release_rust.yml

@@ -38,6 +38,10 @@ jobs:
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: actions/checkout@v3
       - run: |
           git config --global user.email "[email protected]"

.github/workflows/benchmark.yml

@@ -25,6 +25,10 @@ jobs:
         name: Checkout
       - name: Update Rust
         run: rustup update
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - name: Cargo cache
         uses: Swatinem/rust-cache@v2
         with:

.github/workflows/test_python_workflow.yml

@@ -46,6 +46,10 @@ jobs:
           toolchain: 1.77.0
           override: true
           components: rustfmt, clippy
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: webfactory/[email protected]
         name: Load raphtory-disk_graph key
         with:

.github/workflows/test_rust_disk_storage_workflow.yml

@@ -53,6 +53,10 @@ jobs:
         name: Cargo cache
         with:
           cache-all-crates: true
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install nextest
         uses: taiki-e/install-action@nextest
       - name: Activate pometry-storage in Cargo.toml

.github/workflows/test_rust_workflow.yml

@@ -14,11 +14,12 @@ jobs:
     runs-on: '${{ matrix.os }}'
     env:
       CARGO_NET_GIT_FETCH_WITH_CLI: true
+      RUST_BACKTRACE: 1
     strategy:
       matrix:
         include:
           - os: macos-latest
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
           - os: windows-latest
     steps:
       - uses: maxim-lobanov/setup-xcode@v1
@@ -36,12 +37,16 @@ jobs:
           override: true
           components: rustfmt, clippy
       - name: Free up space (ubuntu)
-        if: matrix.os == 'ubuntu-latest'
+        if: "contains(matrix.os, 'ubuntu')"
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/ghc
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: webfactory/[email protected]
         name: Load pometry-storage key
         with:
@@ -78,6 +83,10 @@ jobs:
           toolchain: 1.77.0
           override: true
           components: rustfmt, clippy
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}        
       - uses: Swatinem/rust-cache@v2
         name: Cargo cache
         with:

Cargo.toml

@@ -127,6 +127,11 @@ url = "2.2"
 base64-compat = { package = "base64-compat", version = "1.0.0" }
 time = "0.3.36"
 
+prost = "0.12"
+prost-types = "0.12"
+bytes = "1.6.0"
+prost-build = "0.12"
+
 lazy_static = "1.4.0"
 pest = "2.7.8"
 pest_derive = "2.7.8"

python/Cargo.toml

@@ -19,7 +19,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { workspace = true }
-raphtory_core = { path = "../raphtory", version = "0.9.3", features = ["python", "search", "vectors"], package = "raphtory" }
+raphtory_core = { path = "../raphtory", version = "0.9.3", features = ["python", "search", "vectors", "proto"], package = "raphtory" }
 raphtory-graphql = { path = "../raphtory-graphql", version = "0.9.3",features = ["python"] }
 serde_json = { workspace = true }
 reqwest = { workspace = true }

python/tests/test_graphdb.py

@@ -19,6 +19,7 @@
 import os
 import shutil
 import numpy as np
+import pickle
 
 base_dir = Path(__file__).parent
 edges = [(1, 1, 2), (2, 1, 3), (-1, 2, 1), (0, 1, 1), (7, 3, 2), (1, 1, 1)]
@@ -59,6 +60,29 @@ def test_graph_len_edge_len():
     assert g.count_edges() == 5
 
 
+def test_graph_pickle():
+    g = create_graph()
+    # pickle graph
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        pickle.dump(g, open(tmpdirname + "/graph.p", "wb"))
+        # unpickle graph
+        g2 = pickle.load(open(tmpdirname + "/graph.p", "rb"))
+
+        assert g2.count_nodes() == 3
+        assert g2.count_edges() == 5
+
+
+def test_persistent_graph_pickle():
+    g = create_graph_with_deletions()
+    # pickle graph
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        pickle.dump(g, open(tmpdirname + "/graph.p", "wb"))
+        # unpickle graph
+        g2 = pickle.load(open(tmpdirname + "/graph.p", "rb"))
+
+        assert g2.count_nodes() == 3
+        assert g2.count_edges() == 5
+
 def test_id_iterable():
     g = create_graph()
 

raphtory-graphql/src/lib.rs

@@ -53,7 +53,10 @@ mod graphql_test {
     };
     use serde_json::json;
     use std::collections::{HashMap, HashSet};
-    use tempfile::{tempdir, TempDir};
+    use tempfile::tempdir;
+
+    #[cfg(feature = "storage")]
+    use tempfile::TempDir;
 
     #[tokio::test]
     async fn search_for_gandalf_query() {
@@ -266,7 +269,6 @@ mod graphql_test {
 
         let req = Request::new(prop_has_key_filter);
         let res = schema.execute(req).await;
-        let data = res.data.into_json().unwrap();
         let expected = json!({
             "graph": {
               "properties": {

raphtory/Cargo.toml

@@ -67,8 +67,6 @@ polars-arrow = { workspace = true, optional = true }
 polars-parquet = { workspace = true, optional = true }
 polars-utils = { workspace = true, optional = true }
 kdam = { workspace = true, optional = true }
-
-# disk storage optional dependencies
 memmap2 = { workspace = true, optional = true }
 ahash = { workspace = true, optional = true }
 tempfile = { workspace = true, optional = true }
@@ -77,6 +75,10 @@ rpds = { workspace = true, optional = true }
 thread_local = { workspace = true, optional = true }
 pometry-storage = { workspace = true, optional = true }
 
+prost = { workspace = true, optional = true}
+prost-types = { workspace = true, optional = true}
+bytes = { workspace = true, optional = true}
+
 [dev-dependencies]
 csv = { workspace = true }
 pretty_assertions = { workspace = true }
@@ -89,6 +91,9 @@ dotenv = { workspace = true }            # for vector testing
 streaming-stats = { workspace = true }
 proptest = { workspace = true }
 
+[build-dependencies]
+prost-build = { workspace = true, optional = true}
+
 [features]
 default = []
 # Enables the graph loader io module
@@ -113,6 +118,7 @@ python = [
     "arrow",
     "search",
     "vectors",
+    "proto",
     "dep:pyo3",
     "dep:num",
     "dep:display-error-chain",
@@ -142,3 +148,11 @@ arrow = [
     "polars-parquet?/compression",
     "dep:kdam",
 ]
+
+proto = [
+    "dep:prost",
+    "dep:prost-types",
+    "dep:bytes",
+    "dep:prost-build",
+    "dep:memmap2",
+]

raphtory/build.rs

@@ -0,0 +1,11 @@
+use std::io::Result;
+#[cfg(feature = "proto")]
+fn main() -> Result<()> {
+    prost_build::compile_protos(&["src/graph.proto"], &["src/"])?;
+    Ok(())
+}
+
+#[cfg(not(feature = "proto"))]
+fn main() -> Result<()> {
+    Ok(())
+}

raphtory/src/core/utils/errors.rs

@@ -132,6 +132,14 @@ pub enum GraphError {
 
     #[error("Illegal set error {0}")]
     IllegalSet(String),
+
+    #[cfg(feature = "proto")]
+    #[error("Protobuf encode error{0}")]
+    DecodeError(#[from] prost::DecodeError),
+
+    #[cfg(feature = "proto")]
+    #[error("Protobuf decode error{0}")]
+    EncodeError(#[from] prost::EncodeError),
 }
 
 impl GraphError {