fix evenloop and add neo4j

.gitignore

@@ -5,3 +5,5 @@ book.txt
 lightrag-dev/
 .idea/
 dist/
+/.lightRagEnv
+volumes/

docker-compose.yml

@@ -0,0 +1,83 @@
+version: '3.5'
+
+services:
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.5
+    environment:
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2023-03-20T20-16-18Z
+    environment:
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    ports:
+      - "9001:9001"
+      - "9000:9000"
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+    command: minio server /minio_data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  standalone:
+    container_name: milvus-standalone
+    image: milvusdb/milvus:v2.4.13-hotfix
+    command: ["milvus", "run", "standalone"]
+    security_opt:
+      - seccomp:unconfined
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      interval: 30s
+      start_period: 90s
+      timeout: 20s
+      retries: 3
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+      - "minio"
+
+  neo4j:
+    image: neo4j:latest
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/neo4j/logs:/logs
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/neo4j/config:/config
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/neo4j/data:/data
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/neo4j/plugins:/plugins
+    environment:
+      - NEO4J_AUTH=neo4j/admin12345
+      - NEO4JLABS_PLUGINS=["graph-data-science"]
+      - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes
+      - NEO4J_dbms_security_procedures_unrestricted=gds.*
+      - NEO4J_dbms_security_procedures_allowlist=gds.*
+    ports:
+      - "7474:7474"
+      - "7687:7687"
+    restart: always
+
+networks:
+  default:
+    name: milvus

examples/lightrag_openai_demo_storage.py

@@ -0,0 +1,43 @@
+import os
+
+from lightrag import LightRAG, QueryParam
+from lightrag.llm import gpt_4o_mini_complete
+from lightrag.storage import Neo4jKVStorage
+# WORKING_DIR = "./dickens"
+
+# if not os.path.exists(WORKING_DIR):
+#     os.mkdir(WORKING_DIR)
+
+rag = LightRAG (
+    working_dir="./dickens",
+    llm_model_func=gpt_4o_mini_complete,
+    neo4j_config={
+        "uri": "neo4j://localhost:7687",
+        "username": "neo4j",
+        "password": "admin12345"
+    },
+)
+
+
+with open("./book.txt", "r", encoding="utf-8") as f:
+    rag.insert(f.read())
+
+# Perform naive search
+print(
+    rag.query("What are the top themes in this story?", param=QueryParam(mode="naive"))
+)
+
+# Perform local search
+print(
+    rag.query("What are the top themes in this story?", param=QueryParam(mode="local"))
+)
+
+# Perform global search
+print(
+    rag.query("What are the top themes in this story?", param=QueryParam(mode="global"))
+)
+
+# Perform hybrid search
+print(
+    rag.query("What are the top themes in this story?", param=QueryParam(mode="hybrid"))
+)

lightrag/lightrag.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 from functools import partial
 from typing import Type, cast
-
+from .storage import Neo4jKVStorage, Neo4jGraphStorage
 from .llm import (
     gpt_4o_mini_complete,
     openai_embedding,
@@ -44,9 +44,8 @@ def always_get_an_event_loop() -> asyncio.AbstractEventLoop:
     try:
         loop = asyncio.get_running_loop()
     except RuntimeError:
-        logger.info("Creating a new event loop in a sub-thread.")
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
+        logger.info("Creating a new event loop in the main thread.")
+        loop = asyncio.get_event_loop()
     return loop
 
 
@@ -97,23 +96,34 @@ class LightRAG:
     vector_db_storage_cls_kwargs: dict = field(default_factory=dict)
     graph_storage_cls: Type[BaseGraphStorage] = NetworkXStorage
     enable_llm_cache: bool = True
-
+
+    # Neo4j configuration
+    neo4j_config: dict = field(default_factory=dict)
+    milvus_config: dict = field(default_factory=dict)
+
     # extension
     addon_params: dict = field(default_factory=dict)
     convert_response_to_json_func: callable = convert_response_to_json
 
     def __post_init__(self):
+        if not os.path.exists(self.working_dir):
+            os.makedirs(self.working_dir)
+            logger.info(f"Creating working directory {self.working_dir}")
         log_file = os.path.join(self.working_dir, "lightrag.log")
         set_logger(log_file)
         logger.info(f"Logger initialized for working directory: {self.working_dir}")
 
         _print_config = ",\n  ".join([f"{k} = {v}" for k, v in asdict(self).items()])
         logger.debug(f"LightRAG init with param:\n  {_print_config}\n")
-
-        if not os.path.exists(self.working_dir):
-            logger.info(f"Creating working directory {self.working_dir}")
-            os.makedirs(self.working_dir)
-
+
+        if self.neo4j_config.get("uri") and self.neo4j_config.get("username") and self.neo4j_config.get("password"):
+            self.key_string_value_json_storage_cls = Neo4jKVStorage
+            self.graph_storage_cls = Neo4jGraphStorage
+            logger.info("Using Neo4jKVStorage")
+        else:
+            self.key_string_value_json_storage_cls = JsonKVStorage
+            logger.info("Using JsonKVStorage")
+
         self.full_docs = self.key_string_value_json_storage_cls(
             namespace="full_docs", global_config=asdict(self)
         )